vjkBiXRQ

1import sys
2import urllib
3import urllib2
4import json
5import sqlite3
6from collections import namedtuple
7
8Entry = namedtuple('Entry', 'name position allocation gross_pay net_pay active')
9
10url_opener = urllib2.build_opener()
11url_opener.addheaders = [('User-Agent', 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10136')]
12
13url = 'http://www.tjsp.jus.br/RHF/PortalTransparenciaAPI/sema/FolhaPagamentoMagistrado/ListarTodosPorCargoMesAno'
14max_results = 10000
15
16create_table = '''create table if not exists paychecks(
17                    year integer,
18                    month integer,
19                    source text,
20                    name text,
21                    position text,
22                    allocation text,
23                    gross_pay real,
24                    net_pay real,
25                    active integer,
26                    unique(year, month, source, name, active))'''
27
28def post_data(year, month, active):
29    return { 'draw': '1',
30             'columns[0][data]': 'nome',
31             'columns[0][name]': '',
32             'columns[0][searchable]': 'true',
33             'columns[0][orderable]': 'false',
34             'columns[0][search][value]': '',
35             'columns[0][search][regex]': 'false',
36             'columns[1][data]': 'lotacao',
37             'columns[1][name]': '',
38             'columns[1][searchable]': 'true',
39             'columns[1][orderable]': 'false',
40             'columns[1][search][value]': '',
41             'columns[1][search][regex]': 'false',
42             'columns[2][data]': 'folhaMagistradoCargo.descricao',
43             'columns[2][name]': '',
44             'columns[2][searchable]': 'true',
45             'columns[2][orderable]': 'false',
46             'columns[2][search][value]': '',
47             'columns[2][search][regex]': 'false',
48             'columns[3][data]': 'totalCredito',
49             'columns[3][name]': '',
50             'columns[3][searchable]': 'true',
51             'columns[3][orderable]': 'false',
52             'columns[3][search][value]': '',
53             'columns[3][search][regex]': 'false',
54             'columns[4][data]': 'totalDebitos',
55             'columns[4][name]': '',
56             'columns[4][searchable]': 'true',
57             'columns[4][orderable]': 'false',
58             'columns[4][search][value]': '',
59             'columns[4][search][regex]': 'false',
60             'columns[5][data]': 'rendimentoLiquido',
61             'columns[5][name]': '',
62             'columns[5][searchable]': 'true',
63             'columns[5][orderable]': 'false',
64             'columns[5][search][value]': '',
65             'columns[5][search][regex]': 'false',
66             'columns[6][data]': 'remuneracaoOrgaoOrigem',
67             'columns[6][name]': '',
68             'columns[6][searchable]': 'true',
69             'columns[6][orderable]': 'false',
70             'columns[6][search][value]': '',
71             'columns[6][search][regex]': 'false',
72             'columns[7][data]': 'diarias',
73             'columns[7][name]': '',
74             'columns[7][searchable]': 'true',
75             'columns[7][orderable]': 'false',
76             'columns[7][search][value]': '',
77             'columns[7][search][regex]': 'false',
78             'columns[8][data]': '',
79             'columns[8][name]': '',
80             'columns[8][searchable]': 'true',
81             'columns[8][orderable]': 'false',
82             'columns[8][search][value]': '',
83             'columns[8][search][regex]': 'false',
84             'start': 0,
85             'length': max_results,
86             'search[value]': '',
87             'search[regex]': 'false',
88             'mes': month,
89             'ano': year,
90             'ativo': ['false', 'true'][int(active)],
91             'cargoId': '',
92             'nome': '' }
93
94def fetch(year, month):
95    result = []
96    for active in False, True:
97        values = post_data(year, month, active)
98        data = urllib.urlencode(values)
99        try:
100            response = url_opener.open(url, data)
101        except urllib2.HTTPERROR: 
102            print 'Failed to download %s' % url
103            break
104        data = json.loads(response.read())
105        for e in data['data']:
106            result.append(Entry(
107                name = e['nome'],
108                position = e['folhaMagistradoCargo']['descricao'],
109                allocation = e['lotacao'],
110                gross_pay = float(e['totalCredito']),
111                net_pay = float(e['rendimentoLiquido']),
112                active = int(active)))
113    return result
114
115def main(argv):
116    try:
117        year, month = int(argv[1]), int(argv[2])
118    except:
119        print "Usage: python fetcher.py <year> <month>"
120        return 1
121
122    db = sqlite3.connect('tjsp.db')
123    cursor = db.cursor()
124
125    cursor.execute(create_table)
126
127    result = fetch(year, month)
128    print "Got %d entries" % len(result)
129
130    values = [(year, month, 'TJSP', e.name, e.position, e.allocation, e.gross_pay, e.net_pay, int(e.active)) for e in result]
131    cursor.executemany('insert into paychecks values (?,?,?,?,?,?,?,?,?)', values)
132    db.commit()
133
134if __name__ == "__main__":
135    sys.exit(main(sys.argv))