i8ZALT9k

1# coding: utf-8
2# author xmpp: p(eaZ@exploit.im
3
4from grab.spider import Spider, Task
5from threading import Thread, RLock
6from Queue import Queue
7
8import sqlite3
9import logging
10import md5
11import sys
12import re
13
14# CONFIG
15# 'SELL' or 'BUY' or 'BOTH'
16PAGES_TYPE  = 'BOTH' 
17
18# number of pages from first or 'ALL'
19PAGES_COUNT = 'ALL' 
20
21# threads not recommend > 20 (DoS)
22THREADS = 20
23
24# results database file name
25RESULTS_FILE_NAME = 'results.db'
26# CONFIG END
27DEBUG = False
28
29
30class DB(Thread):
31
32    def __init__(self):
33        Thread.__init__(self)
34        self.setDaemon = True
35        self.lock = RLock()
36        self._queries = Queue()
37
38        self._con = sqlite3.connect(
39            RESULTS_FILE_NAME, 
40            check_same_thread=False
41        )
42
43        self._con.text_factory = str
44        self._cur = self._con.cursor()
45
46        self._cur.execute("""
47            CREATE TABLE IF NOT EXISTS phones (
48                phone VARCHAR(100) PRIMARY KEY,
49                diller VARCHAR(100)
50                )
51            """)
52
53        self._cur.execute("""
54            CREATE TABLE IF NOT EXISTS parsed_urls (
55                hash VARCHAR(100) PRIMARY KEY,
56                url VARCHAR(100)
57                )
58            """)
59
60        self._cur.execute(
61            "CREATE INDEX IF NOT EXISTS parsed_urls_idx ON parsed_urls(hash)"
62            )
63
64        self._con.commit()
65
66    def execute(self, query):
67        self._queries.put(query)
68
69    def fetch(self, query):
70        with self.lock:
71            return self._cur.execute(query).fetchall()
72
73    def run(self):
74        while 1:
75            query = self._queries.get()
76            if query == 'exit':
77                return
78            if query:
79                try:
80                    with self.lock:
81                        self._cur.execute(query)
82                        self._con.commit()
83                except:
84                    pass
85        self._con.close()
86
87    def exit(self):
88        self._queries.put('exit')
89
90
91class SubitoSpider(Spider):
92
93    def task_initial(self, grab, task):
94        ultima_link = grab.doc.select(
95            '//*/div[@class="pagination_bottom_link"]/a/@href'
96        ).text()
97
98        page_nums = re.findall('o=(\d+)', ultima_link)
99        if page_nums:
100            page_nums = int(page_nums[0])
101
102        if self.config.get('pages_count') != 'ALL':
103            page_nums = int(self.config.get('pages_count'))
104
105        for page_num in xrange(page_nums):
106            page_url = grab.doc.url + '&o=' + str(page_num)
107            self.add_task(Task('get_ads_urls', url=page_url))
108
109    def task_get_ads_urls(self, grab, task):
110        ad_nodes = grab.doc.select(
111            '//*/div[@class="item_list_inner"]'
112        ).node_list()
113
114   
115        for ad_node in ad_nodes:
116            ad_url = ad_node.xpath('.//h2/a/@href')[0]
117
118            url_hash = make_md5(ad_url)
119            if not db.fetch('SELECT * FROM parsed_urls WHERE hash="%s"' % url_hash):
120                db.execute('INSERT INTO parsed_urls(hash, url) VALUES ("%s", "%s")' % (url_hash, ad_url))
121                self.add_task(Task('parse_ad', url=ad_url))
122
123    def task_parse_ad(self, grab, task):
124        try:
125            phone = grab.doc.select(
126                '//*/span[@id="adv_phone_full"]'
127            ).text().strip().encode('utf-8')
128
129            diller = grab.doc.select(
130                '//*/strong[@class="author btn_author_reply"]'
131            ).text().strip().encode('utf-8')
132        except:
133            return
134
135        if  phone.startswith('3') \
136        and len(phone) >= 9 <= 10:
137            query = 'INSERT INTO phones(phone, diller) VALUES ("%s", "%s")' % (phone, diller)
138            db.execute(query)
139            logging.info('FOUND {} - {}'.format(phone, diller))
140
141
142def make_md5(string):
143    m = md5.new()
144    m.update(string)
145    return str(m.hexdigest())
146
147
148if __name__ == '__main__':
149
150    if DEBUG:
151        logging.basicConfig(level=logging.DEBUG)
152    else:
153        logging.basicConfig(level=logging.INFO)
154
155    db = DB()
156    db.start()
157
158    bot = SubitoSpider(thread_number=THREADS)
159    bot.config['pages_count'] = PAGES_COUNT
160
161
162    if PAGES_TYPE == 'SELL':
163        bot.initial_urls = ['https://www.subito.it/annunci-italia/vendita/usato/?qso=true']
164    elif PAGES_TYPE == 'BUY':
165        bot.initial_urls = ['https://www.subito.it/annunci-italia/cerco/usato/?qso=true']
166    elif PAGES_TYPE == 'BOTH':
167        bot.initial_urls = [
168            'https://www.subito.it/annunci-italia/vendita/usato/?qso=true',
169            'https://www.subito.it/annunci-italia/cerco/usato/?qso=true',
170        ]
171    else:
172        sys.exit('Unknown pages type: %s' % PAGES_TYPE)
173
174    try:
175       bot.run()
176    except KeyboardInterrupt:
177       bot.stop()
178    finally:
179        db.exit()