· 9 years ago · Jan 31, 2017, 06:56 PM
1'''
25902a57cf13f6280665357eaceec1cabe9129f91369475697abe85be52278ea8
3
4
5In this step we'll be creating a distributed index. This script is comprised of
6a scanner called the Carpenter that will spawn several threads, each sending out
7feelers looking for web servers out there that are willing to respond. Responses
8are kept in a sqlite database named walrus.db
9
10walrus.db will contain a table, Walrus, with the following columns:
11 host -- The host IP address of the discovered server
12 key -- A header key from the server response
13 val -- The associated value to the header key
14
15This will allow us to later create a massively parallel database of all servers
16that interest us. But for now, let's just do some collecting.
17
18Save this file as collect.py and execute it using
19python (https://www.python.org/downloads/) with the following command:
20
21python collect.py
22
23The longer you run this the more you will collect. You can quit and resume at
24any time.
25
26Feel free to query this table directly should you be interested in this
27intermediate state. But whatever you do, keep your walrus.db file safe because
28we'll be using this in our next step, which will be published with the following
29key: 38a11b99402ab435eb0bc53419bf278e582cb3c3361da5be1b8f69d3bf422470
30
31Thank you for your support. Together we can do this!
32
33DISCLAIMER: Your ISP (or local law) may prohibit scanning of this nature. Hey,
34let's be careful out there.
35'''
36
37try:
38 from Queue import Queue
39except:
40 from queue import Queue
41from random import randrange as rand
42try:
43 from requests import get
44except:
45 print('requests module required:\npip install requests')
46 exit()
47from sqlite3 import connect
48from threading import Thread, active_count
49from time import sleep
50
51# whitelist of header keys
52KEYS = [
53 'connection',
54 'content-encoding',
55 'content-length',
56 'content-type',
57 'etag',
58 'server',
59 'x-frame-options',
60 'x-powered-by'
61]
62
63class Carpenter:
64
65 def __init__(self, threadcount=100):
66 self.threadcount = threadcount
67
68 def run(self):
69 '''
70 Let's get this party started!
71 '''
72 queue = Queue()
73 threads = []
74 for i in range(self.threadcount):
75 self.walk(self.worker, queue)
76 self.walk(self.tweedle, queue)
77 while active_count() > 0:
78 sleep(0.25)
79
80 def walk(self, fn, queue):
81 '''
82 Spawn a daemon thread. 2spooky4me.
83 '''
84 thread = Thread(target=fn, args=(queue,))
85 thread.daemon = True
86 thread.start()
87
88 def tweedle(self, queue):
89 '''
90 Process the queue in one thread (because sqlite).
91 '''
92 dee = connect('walrus.db')
93 dum = dee.cursor()
94 dum.execute('''
95 create table if not exists Walrus (host text, key text, val text)
96 ''')
97 dum.execute('create index if not exists idx_host on Walrus (host)')
98 dee.commit()
99 while True:
100 host, headers = queue.get()
101 if headers:
102 print(host)
103 values = []
104 for key, val in headers.items():
105 print(' ', key, val)
106 values.append((host, key, val))
107 dum.executemany('''
108 insert into Walrus (host, key, val) values (?, ?, ?)
109 ''', values)
110 dee.commit()
111
112 def work(self):
113 '''
114 Do work on one random host.
115 '''
116 host = '{}.{}.{}.{}'.format(rand(256), rand(256), rand(256), rand(256))
117 req = get('http://' + host, timeout=5, headers={
118 'User-Agent': 'Do you admire the view?',
119 })
120 headers = {}
121 for key, val in req.headers.items():
122 # Restrict the value length... There are weirdos ones out there...
123 if key.lower() in KEYS and len(val) < 256:
124 headers[key] = val
125 headers['status'] = req.status_code
126 return [host, headers]
127
128 def worker(self, queue):
129 '''
130 Work forever... :|
131 Don't worry, computers are into that kind of thing.
132 '''
133 while True:
134 try:
135 result = self.work()
136 except:
137 continue
138 queue.put(result)
139
140
141if __name__ == '__main__':
142 Carpenter().run()