free_adsl_bill_fetcher 10.4 KB
Newer Older
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
1
#!/usr/bin/env python
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
2
# -*- coding: utf-8 -*-
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
#
# Copyright (c) Remi Ferrand <remi.ferrand_at_riton.fr>, 2014
# 
# This software is a computer program whose purpose is to [describe
# functionalities and technical features of your software].
# 
# This software is governed by the CeCILL  license under French law and
# abiding by the rules of distribution of free software.  You can  use, 
# modify and/ or redistribute the software under the terms of the CeCILL
# license as circulated by CEA, CNRS and INRIA at the following URL
# "http://www.cecill.info". 
# 
# As a counterpart to the access to the source code and  rights to copy,
# modify and redistribute granted by the license, users are provided only
# with a limited warranty  and the software's author,  the holder of the
# economic rights,  and the successive licensors  have only  limited
# liability. 
# 
# In this respect, the user's attention is drawn to the risks associated
# with loading,  using,  modifying and/or developing or reproducing the
# software by the user in light of its specific status of free software,
# that may mean  that it is complicated to manipulate,  and  that  also
# therefore means  that it is reserved for developers  and  experienced
# professionals having in-depth computer knowledge. Users are therefore
# encouraged to load and test the software's suitability as regards their
# requirements in conditions enabling the security of their systems and/or 
# data to be ensured and,  more generally, to use and operate it in the 
# same conditions as regards security. 
# 
# The fact that you are presently reading this means that you have had
# knowledge of the CeCILL license and that you accept its terms.
#
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
35

Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
36
import re
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
37 38 39
import os
import sys
import json
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
40
import logging
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
41
import requests
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
42
import HTMLParser
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
43
from optparse import OptionParser
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
44
from prettytable import PrettyTable
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
45 46


Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
47 48 49 50 51 52 53 54 55 56 57 58
# try:
#     import http.client as http_client
# except ImportError:
#         import httplib as http_client
# 
# http_client.HTTPConnection.debuglevel = 1
# 
# logging.basicConfig()
# logging.getLogger().setLevel(logging.DEBUG)
# requests_log = logging.getLogger("requests.packages.urllib3")
# requests_log.setLevel(logging.DEBUG)
# requests_log.propagate = True
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
59

Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
60
class HTTPClient(object):
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
61 62

    USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36'
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
    
    def __init__(this):
        pass

    def _default_headers(this):
        return {
            'User-Agent': this.USER_AGENT
        }

    def get(this, url, headers={}, *args, **kwargs):
        h = this._default_headers().copy()
        headers.update(h)
        return requests.get(url,
                headers=headers,
                *args, **kwargs)

    def post(this, url, headers={}, *args, **kwargs):
        h = this._default_headers().copy()
        headers.update(h)

        return requests.post(url,
                headers=headers,
                *args, **kwargs)
    
    @staticmethod
    def unescapeHTML(text):
        return HTMLParser.HTMLParser().unescape(text)


class FreeAdslBillFetcher(HTTPClient):

Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
94 95
    LOGIN_URL = 'https://subscribe.free.fr/login/login.pl'
    LOGOUT_URL = 'https://adsl.free.fr/logout.pl'
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
96 97 98 99
    BILL_ROOT_URL = 'https://adsl.free.fr'
    LIST_BILLS_URL = BILL_ROOT_URL + '/liste-factures.pl'

    BILLFINDER_RE = re.compile(r'<strong>(?P<month>\w+?\s*\d{4})</strong>.*?<strong>(?P<price>\d+(?:\.\d+)?) Euros</strong>.*?<a href="(?P<url>facture_pdf.pl.+?)"', re.S | re.UNICODE)
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
100 101 102 103 104 105 106 107 108 109 110

    class FreeSessionCredentials(object):
        def __init__(this, id, idt):
            this.id = id
            this.idt = idt

        def toDict(this):
            return {'id': this.id, 'idt': this.idt}

        def __str__(this):
            return "id=%s&idt=%s" % (this.id, this.idt)
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
111

Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
112 113 114 115 116 117 118 119 120 121 122 123 124
    class FreeAdslBill(HTTPClient):
        def __init__(this, title, amount, url):
            this.title = title
            this.amount = amount
            this.url = url

        def __eq__(this, other):
            return this.title == other.title

        def __repr__(this):
            return "FreeAdslBill<%s>" % this.title.encode('utf-8')


Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
125 126 127 128 129
    def __init__(this, user, password):

        this.user = user
        this.password = password

Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
130
        this._transaction_creds = None
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
131 132


Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
133
    def login(this):
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
134 135
        payload = {
            'login': this.user,
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
136 137 138
            'pass' : this.password,
            'ok'   : 'Valider',
            'link' : ''
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
139 140
        }

Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
141
        headers = {
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
142
            'Content-Type': 'application/x-www-form-urlencoded'
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
143
        }
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
144

Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
145
        r = this.post(this.LOGIN_URL, headers=headers, data=payload,
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
146 147 148 149 150 151 152 153 154
                allow_redirects=False)

        location = r.headers['location']
        this._buildSessionCreds(location)

    def _buildSessionCreds(this, url):
        # https://adsl.free.fr/home.pl?id=16779576&idt=1015c62833046b0f
        ids_s = url[url.index('?')+1:]
        h = dict(x.split('=') for x in ids_s.split('&'))
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
155
        this._transaction_creds = FreeAdslBillFetcher.FreeSessionCredentials(id=h['id'], idt=h['idt'])
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
156 157 158

    def _appendUrlCreds(this, url):
        creds = this._transaction_creds
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
159
        return url + '?' + str(creds)
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
160 161 162 163 164 165 166

    def listBills(this):
        if this._transaction_creds is None: this.login()

        # We need to build the URL/params by hand.
        # python-requests params doesn't work here
        # it seems that url parameters are order dependant
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
167
        # on this webservice
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
168
        url = this._appendUrlCreds(this.LIST_BILLS_URL)
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
169
        r = this.get(url, allow_redirects=False)
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
170

Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
171 172 173 174 175 176 177 178 179 180 181
        bills = this._parseBillsList(this.unescapeHTML(r.text))
        return bills

    def fetchBill(this, bill):
        # assume that whole PDF fits in memory
        return this.get(bill.url).content

    def writeBillToFile(this, bill, file):
        # assume that whole PDF fits in memory
        with open(file, 'wb') as f:
            f.write(this.fetchBill(bill))
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
182 183 184


    def _parseBillsList(this, body):
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
185 186 187 188 189 190 191
        match = this.BILLFINDER_RE.findall(body)

        bills = []
        for month, price, url in match:
            bills.append(FreeAdslBillFetcher.FreeAdslBill(month, float(price), this.BILL_ROOT_URL + '/' + url))

        return bills
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
192 193 194 195 196

    def logout(this):
        if this._transaction_creds is None:
            return
        url = this._appendUrlCreds(this.LOGOUT_URL)
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
197
        this.get(url, allow_redirects=False)
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
198 199 200 201 202 203 204
        
    def __enter__(this):
        this.login()
        return this

    def __exit__(this, type, value, traceback):
        this.logout()
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
205

Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
206 207
class FreeAdslBillFetcherCli(object):

Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
208
    PROG_NAME = 'free_adsl_bill_fetcher'
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248

    @staticmethod
    def _buildOptParser():
        parser = OptionParser()
        parser.add_option("-p", "--show-price", dest="show_price", default=False,
                action="store_true", help="show price when listing bills")
        parser.add_option("--latest", dest="fetch_latest", default=False,
                action="store_true", help="only fetch latest bill")
        parser.add_option("-c", "--config", dest="config_file",
                default=os.path.expanduser('~') + '/.' + FreeAdslBillFetcherCli.PROG_NAME + '.conf',
                metavar='FILE', help="configuration file")
        parser.add_option("--get", dest="wanted_bills",
                action="append",
                default=[],
                metavar='BILL_TITLE', help="Download bill BILL_TITLE and write it as BILL_TITLE.pdf")
        parser.add_option("-d", "--write-dir", dest="write_directory",
                default='.',
                metavar='DIR', help="write bills to directory DIR")
        parser.add_option("--name-prefix", dest="name_prefix",
                default='',
                metavar='STR', help="prefix each bill filename with STR")
        parser.add_option("--name-suffix", dest="name_suffix",
                default='',
                metavar='STR', help="suffix each bill filename with STR (before PDF extension)")

        return parser

    def parseArgs(this, arg):
        (options, args) = this.opt_parser.parse_args(args=arg)
        this.options = options
        this.args = args

    def __init__(this, args=sys.argv[1:]):
        this.opt_parser = this._buildOptParser()
        this.options = None
        this.args = None
        this.parseArgs(args)

        this.config = this._parseConfigFile(this.options.config_file)
        this.fetcher = None
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
249
        this.table = this._prepareTable()
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
250 251


Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
252 253 254 255 256 257 258 259 260 261 262 263 264
    def __enter__(this, *args, **kwargs):
        this.fetcher = FreeAdslBillFetcher(this.config['login'], this.config['password'])
        this.fetcher.login()
        return this

    def __exit__(this, type, value, traceback):
        this.fetcher.logout()

    @staticmethod
    def _parseConfigFile(file):
        with open(file) as f:
            return json.load(f)

Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
265 266 267 268 269 270 271 272
    def _prepareTable(this):
        headers = ['Month']
        if this.options.show_price:
            headers.insert(1, 'Price')
        pt = PrettyTable(headers)
        pt.align['Month'] = 'l'
        return pt

Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
273
    def run(this):
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
274

Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
275
        wanted_bills = len(this.options.wanted_bills)
276

Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
277
        if wanted_bills != 0 or this.options.fetch_latest:
278 279 280
            table_output = False
        else:
            table_output = True
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
281 282 283 284 285 286 287 288 289 290 291

        for bill in this.fetcher.listBills():

            if this.options.fetch_latest is True:
                pdf_path = this._fetchBill(bill)
                print "Your latest bill was written to '%s'" % pdf_path
                break

            if wanted_bills > 0:
                if bill.title in [x.decode('utf-8') for x in this.options.wanted_bills]:
                    pdf_path = this._fetchBill(bill)
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
292
                    print "[*] %s bill was written to '%s'" % (bill.title, pdf_path)
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
293 294

            else:
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
295 296 297 298
                this._appendBillToTable(bill)

        if table_output:
            print this.table
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
299 300 301

        sys.exit(0)

Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
302 303
    def _appendBillToTable(this, bill):
        row = [bill.title]
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
304
        if this.options.show_price:
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
305 306
            row.insert(1, '%.2f €' % bill.amount)
        this.table.add_row(row)
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
307 308 309 310 311 312 313 314 315 316 317 318

    def _composeBillFilename(this, bill):
        bill_filename = this.options.write_directory.rstrip('/') + '/'
        bill_filename += this.options.name_prefix
        bill_filename += re.sub(r'\s+', '_', bill.title)
        bill_filename += this.options.name_suffix + '.pdf'
        return bill_filename

    def _fetchBill(this, bill):
        bill_filename = this._composeBillFilename(bill)
        this.fetcher.writeBillToFile(bill, bill_filename)
        return bill_filename
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
319 320

if __name__ == "__main__":
Rémi FERRAND ⛰'s avatar
Rémi FERRAND ⛰ committed
321 322
    with FreeAdslBillFetcherCli() as cli:
        cli.run()