· 9 years ago · Oct 20, 2016, 07:12 PM
1#!/usr/bin/env python
2#
3#SubBrute v1.2
4#A (very) fast subdomain enumeration tool.
5#
6#Maintained by rook
7#Contributors:
8#JordanMilne, KxCode, rc0r, memoryprint, ppaulojr
9#
10import re
11import optparse
12import os
13import signal
14import sys
15import uuid
16import random
17import ctypes
18import dns.resolver
19import dns.rdatatype
20import json
21
22#Python 2.x and 3.x compatiablity
23#We need the Queue library for exception handling
24try:
25 import queue as Queue
26except:
27 import Queue
28
29#The 'multiprocessing' library does not rely upon a Global Interpreter Lock (GIL)
30import multiprocessing
31
32#Microsoft compatiablity
33if sys.platform.startswith('win'):
34 #Drop-in replacement, subbrute + multiprocessing throws exceptions on windows.
35 import threading
36 multiprocessing.Process = threading.Thread
37
38class verify_nameservers(multiprocessing.Process):
39
40 def __init__(self, target, record_type, resolver_q, resolver_list, wildcards):
41 multiprocessing.Process.__init__(self, target = self.run)
42 self.daemon = True
43 signal_init()
44
45 self.time_to_die = False
46 self.resolver_q = resolver_q
47 self.wildcards = wildcards
48 #Do we need wildcards for other types of records?
49 #This needs testing!
50 self.record_type = "A"
51 if record_type == "AAAA":
52 self.record_type = record_type
53 self.resolver_list = resolver_list
54 resolver = dns.resolver.Resolver()
55 #The domain provided by the user.
56 self.target = target
57 #1 website in the world, modify the following line when this status changes.
58 #www.google.cn, I'm looking at you ;)
59 self.most_popular_website = "www.google.com"
60 #We shouldn't need the backup_resolver, but we we can use them if need be.
61 #We must have a resolver, and localhost can work in some environments.
62 self.backup_resolver = resolver.nameservers + ['127.0.0.1', '8.8.8.8', '8.8.4.4']
63 #Ideally a nameserver should respond in less than 1 sec.
64 resolver.timeout = 1
65 resolver.lifetime = 1
66 try:
67 #Lets test the letancy of our connection.
68 #Google's DNS server should be an ideal time test.
69 resolver.nameservers = ['8.8.8.8']
70 resolver.query(self.most_popular_website, self.record_type)
71 except:
72 #Our connection is slower than a junebug in molasses
73 resolver = dns.resolver.Resolver()
74 self.resolver = resolver
75
76 def end(self):
77 self.time_to_die = True
78
79 #This process cannot block forever, it needs to check if its time to die.
80 def add_nameserver(self, nameserver):
81 keep_trying = True
82 while not self.time_to_die and keep_trying:
83 try:
84 self.resolver_q.put(nameserver, timeout = 1)
85 trace("Added nameserver:", nameserver)
86 keep_trying = False
87 except Exception as e:
88 if type(e) == Queue.Full or str(type(e)) == "<class 'queue.Full'>":
89 keep_trying = True
90
91 def verify(self, nameserver_list):
92 added_resolver = False
93 for server in nameserver_list:
94 if self.time_to_die:
95 #We are done here.
96 break
97 server = server.strip()
98 if server:
99 self.resolver.nameservers = [server]
100 try:
101 #test_result = self.resolver.query(self.most_popular_website, "A")
102 #should throw an exception before this line.
103 if True:#test_result:
104 #Only add the nameserver to the queue if we can detect wildcards.
105 if(self.find_wildcards(self.target)):# and self.find_wildcards(".com")
106 #wildcards have been added to the set, it is now safe to be added to the queue.
107 #blocking queue, this process will halt on put() when the queue is full:
108 self.add_nameserver(server)
109 added_resolver = True
110 trace("Accepted nameserver:", server)
111 else:
112 trace("Rejected nameserver - wildcard:", server)
113 except Exception as e:
114 #Rejected server :(
115 trace("Rejected nameserver - unreliable:", server, type(e))
116 return added_resolver
117
118 def run(self):
119 #Every user will get a different set of resovlers, this helps redistribute traffic.
120 random.shuffle(self.resolver_list)
121 if not self.verify(self.resolver_list):
122 #This should never happen, inform the user.
123 sys.stderr.write('Warning: No nameservers found, trying fallback list.\n')
124 #Try and fix it for the user:
125 self.verify(self.backup_resolver)
126 #End of the resolvers list.
127 try:
128 self.resolver_q.put(False, timeout = 1)
129 except:
130 pass
131
132 #Only add the nameserver to the queue if we can detect wildcards.
133 #Returns False on error.
134 def find_wildcards(self, host):
135 #We want sovle the following three problems:
136 #1)The target might have a wildcard DNS record.
137 #2)The target maybe using geolocaiton-aware DNS.
138 #3)The DNS server we are testing may respond to non-exsistant 'A' records with advertizements.
139 #I have seen a CloudFlare Enterprise customer with the first two conditions.
140 try:
141 #This is case #3, these spam nameservers seem to be more trouble then they are worth.
142 wildtest = self.resolver.query(uuid.uuid4().hex + ".com", "A")
143 if len(wildtest):
144 trace("Spam DNS detected:", host)
145 return False
146 except:
147 pass
148 test_counter = 8
149 looking_for_wildcards = True
150 while looking_for_wildcards and test_counter >= 0 :
151 looking_for_wildcards = False
152 #Don't get lost, this nameserver could be playing tricks.
153 test_counter -= 1
154 try:
155 testdomain = "%s.%s" % (uuid.uuid4().hex, host)
156 wildtest = self.resolver.query(testdomain, self.record_type)
157 #This 'A' record may contain a list of wildcards.
158 if wildtest:
159 for w in wildtest:
160 w = str(w)
161 if w not in self.wildcards:
162 #wildcards were detected.
163 self.wildcards[w] = None
164 #We found atleast one wildcard, look for more.
165 looking_for_wildcards = True
166 except Exception as e:
167 allowed_exceptions = (
168 dns.resolver.NXDOMAIN, # domain which does not exist
169 dns.resolver.NoAnswer, # some resolvers give this result, I dunno why
170 )
171### This is the place where I added the change you fucking bitch
172 if type(e) in allowed_exceptions or type(e) == dns.name.EmptyLabel:
173 #not found
174 return True
175 else:
176 #This resolver maybe flakey, we don't want it for our tests.
177 trace("wildcard exception:", self.resolver.nameservers, type(e))
178 return False
179 #If we hit the end of our depth counter and,
180 #there are still wildcards, then reject this nameserver because it smells bad.
181 return (test_counter >= 0)
182
183class lookup(multiprocessing.Process):
184
185 def __init__(self, in_q, out_q, resolver_q, domain, wildcards, spider_blacklist):
186 multiprocessing.Process.__init__(self, target = self.run)
187 signal_init()
188 self.required_nameservers = 16
189 self.in_q = in_q
190 self.out_q = out_q
191 self.resolver_q = resolver_q
192 self.domain = domain
193 self.wildcards = wildcards
194 self.spider_blacklist = spider_blacklist
195 self.resolver = dns.resolver.Resolver()
196 #Force pydns to use our nameservers
197 self.resolver.nameservers = []
198
199 def get_ns(self):
200 ret = []
201 try:
202 ret = [self.resolver_q.get_nowait()]
203 if ret == False:
204 #Queue is empty, inform the rest.
205 self.resolver_q.put(False)
206 ret = []
207 except:
208 pass
209 return ret
210
211 def get_ns_blocking(self):
212 ret = []
213 ret = [self.resolver_q.get()]
214 if ret == False:
215 trace("get_ns_blocking - Resolver list is empty.")
216 #Queue is empty, inform the rest.
217 self.resolver_q.put(False)
218 ret = []
219 return ret
220
221 def check(self, host, record_type = "A", retries = 0):
222 trace("Checking:", host)
223 cname_record = []
224 retries = 0
225 if len(self.resolver.nameservers) <= self.required_nameservers:
226 #This process needs more nameservers, lets see if we have one avaible
227 self.resolver.nameservers += self.get_ns()
228 #Ok we should be good to go.
229 while True:
230 try:
231 #Query the nameserver, this is not simple...
232 if not record_type or record_type == "A":
233 resp = self.resolver.query(host)
234 #Crawl the response
235 hosts = extract_hosts(str(resp.response), self.domain)
236 for h in hosts:
237 if h not in self.spider_blacklist:
238 self.spider_blacklist[h]=None
239 trace("Found host with spider:", h)
240 self.in_q.put((h, record_type, 0))
241 return resp
242 if record_type == "CNAME":
243 #A max 20 lookups
244 for x in range(20):
245 try:
246 resp = self.resolver.query(host, record_type)
247 except dns.resolver.NoAnswer:
248 resp = False
249 pass
250 if resp and resp[0]:
251 host = str(resp[0]).rstrip(".")
252 cname_record.append(host)
253 else:
254 return cname_record
255 else:
256 #All other records:
257 return self.resolver.query(host, record_type)
258
259 except Exception as e:
260 if type(e) == dns.resolver.NoNameservers:
261 #We should never be here.
262 #We must block, another process should try this host.
263 #do we need a limit?
264 self.in_q.put((host, record_type, 0))
265 self.resolver.nameservers += self.get_ns_blocking()
266 return False
267 elif type(e) == dns.resolver.NXDOMAIN:
268 #"Non-existent domain name."
269 return False
270 elif type(e) == dns.resolver.NoAnswer:
271 #"The response did not contain an answer."
272 if retries >= 1:
273 trace("NoAnswer retry")
274 return False
275 retries += 1
276 elif type(e) == dns.resolver.Timeout:
277 trace("lookup failure:", host, retries)
278 #Check if it is time to give up.
279 if retries >= 3:
280 if retries > 3:
281 #Sometimes 'internal use' subdomains will timeout for every request.
282 #As far as I'm concerned, the authorative name server has told us this domain exists,
283 #we just can't know the address value using this method.
284 return ['Mutiple Query Timeout - External address resolution was restricted']
285 else:
286 #Maybe another process can take a crack at it.
287 self.in_q.put((host, record_type, retries + 1))
288 return False
289 retries += 1
290 #retry...
291 elif type(e) == IndexError:
292 #Some old versions of dnspython throw this error,
293 #doesn't seem to affect the results, and it was fixed in later versions.
294 pass
295 elif type(e) == TypeError:
296 # We'll get here if the number procs > number of resolvers.
297 # This is an internal error do we need a limit?
298 self.in_q.put((host, record_type, 0))
299 return False
300 elif type(e) == dns.rdatatype.UnknownRdatatype:
301 error("DNS record type not supported:", record_type)
302 else:
303 trace("Problem processing host:", host)
304 #dnspython threw some strange exception...
305 raise e
306
307 def run(self):
308 #This process needs one resolver before it can start looking.
309 self.resolver.nameservers += self.get_ns_blocking()
310 while True:
311 found_addresses = []
312 work = self.in_q.get()
313 #Check if we have hit the end marker
314 while not work:
315 #Look for a re-queued lookup
316 try:
317 work = self.in_q.get(blocking = False)
318 #if we took the end marker of the queue we need to put it back
319 if work:
320 self.in_q.put(False)
321 except:#Queue.Empty
322 trace('End of work queue')
323 #There isn't an item behind the end marker
324 work = False
325 break
326 #Is this the end all work that needs to be done?
327 if not work:
328 #Perpetuate the end marker for all threads to see
329 self.in_q.put(False)
330 #Notify the parent that we have died of natural causes
331 self.out_q.put(False)
332 break
333 else:
334 if len(work) == 3:
335 #keep track of how many times this lookup has timedout.
336 (hostname, record_type, timeout_retries) = work
337 response = self.check(hostname, record_type, timeout_retries)
338 else:
339 (hostname, record_type) = work
340 response = self.check(hostname, record_type)
341 sys.stdout.flush()
342 trace(response)
343 #self.wildcards is populated by the verify_nameservers() thread.
344 #This variable doesn't need a muetex, because it has a queue.
345 #A queue ensure nameserver cannot be used before it's wildcard entries are found.
346 reject = False
347 if response:
348 for a in response:
349 a = str(a)
350 if a in self.wildcards:
351 trace("resovled wildcard:", hostname)
352 reject= True
353 #reject this domain.
354 break;
355 else:
356 found_addresses.append(a)
357 if not reject:
358 #This request is filled, send the results back
359 result = (hostname, record_type, found_addresses)
360 self.out_q.put(result)
361
362#Extract relevant hosts
363#The dot at the end of a domain signifies the root,
364#and all TLDs are subs of the root.
365host_match = re.compile(r"((?<=[\s])[a-zA-Z0-9_-]+\.(?:[a-zA-Z0-9_-]+\.?)+(?=[\s]))")
366def extract_hosts(data, hostname):
367 #made a global to avoid re-compilation
368 global host_match
369 ret = []
370 hosts = re.findall(host_match, data)
371 for fh in hosts:
372 host = fh.rstrip(".")
373 #Is this host in scope?
374 if host.endswith(hostname):
375 ret.append(host)
376 return ret
377
378#Return a list of unique sub domains, sorted by frequency.
379#Only match domains that have 3 or more sections subdomain.domain.tld
380domain_match = re.compile("([a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]*)+")
381def extract_subdomains(file_name):
382 #Avoid re-compilation
383 global domain_match
384 subs = {}
385 sub_file = open(file_name).read()
386 f_all = re.findall(domain_match, sub_file)
387 del sub_file
388 for i in f_all:
389 if i.find(".") >= 0:
390 p = i.split(".")[0:-1]
391 #gobble everything that might be a TLD
392 while p and len(p[-1]) <= 3:
393 p = p[0:-1]
394 #remove the domain name
395 p = p[0:-1]
396 #do we have a subdomain.domain left?
397 if len(p) >= 1:
398 trace(str(p), " : ", i)
399 for q in p:
400 if q :
401 #domain names can only be lower case.
402 q = q.lower()
403 if q in subs:
404 subs[q] += 1
405 else:
406 subs[q] = 1
407 #Free some memory before the sort...
408 del f_all
409 #Sort by freq in desc order
410 subs_sorted = sorted(subs.keys(), key = lambda x: subs[x], reverse = True)
411 return subs_sorted
412
413def print_target(target, record_type = None, subdomains = "names.txt", resolve_list = "resolvers.txt", process_count = 16, output = False, json_output = False, found_subdomains=[],verbose=False):
414 subdomains_list = []
415 results_temp = []
416 results = run(target, record_type, subdomains, resolve_list, process_count)
417 #print(type(results))
418 for result in run(target, record_type, subdomains, resolve_list, process_count):
419 (hostname, record_type, response) = result
420 if not record_type:
421 result = hostname
422 else:
423 result = "%s,%s" % (hostname, ",".join(response).strip(","))
424 if result not in found_subdomains:
425 if verbose:
426 print hostname#, record_type, response
427 subdomains_list.append(result)
428
429 return set(subdomains_list)
430
431def run(target, record_type = None, subdomains = "names.txt", resolve_list = "resolvers.txt", process_count = 16):
432 subdomains = check_open(subdomains)
433 resolve_list = check_open(resolve_list)
434 if (len(resolve_list) / 16) < process_count:
435 sys.stderr.write('Warning: Fewer than 16 resovlers per thread, consider adding more nameservers to resolvers.txt.\n')
436 if os.name == 'nt':
437 wildcards = {}
438 spider_blacklist = {}
439 else:
440 wildcards = multiprocessing.Manager().dict()
441 spider_blacklist = multiprocessing.Manager().dict()
442 in_q = multiprocessing.Queue()
443 out_q = multiprocessing.Queue()
444 #have a buffer of at most two new nameservers that lookup processes can draw from.
445 resolve_q = multiprocessing.Queue(maxsize = 2)
446
447 #Make a source of fast nameservers avaiable for other processes.
448 verify_nameservers_proc = verify_nameservers(target, record_type, resolve_q, resolve_list, wildcards)
449 verify_nameservers_proc.start()
450 #The empty string
451 in_q.put((target, record_type))
452 spider_blacklist[target]=None
453 #A list of subdomains is the input
454 for s in subdomains:
455 s = str(s).strip()
456 if s:
457 if s.find(","):
458 #SubBrute should be forgiving, a comma will never be in a url
459 #but the user might try an use a CSV file as input.
460 s=s.split(",")[0]
461 if not s.endswith(target):
462 hostname = "%s.%s" % (s, target)
463 else:
464 #A user might feed an output list as a subdomain list.
465 hostname = s
466 if hostname not in spider_blacklist:
467 spider_blacklist[hostname]=None
468 work = (hostname, record_type)
469 in_q.put(work)
470 #Terminate the queue
471 in_q.put(False)
472 for i in range(process_count):
473 worker = lookup(in_q, out_q, resolve_q, target, wildcards, spider_blacklist)
474 worker.start()
475 threads_remaining = process_count
476 while True:
477 try:
478 #The output is valid hostnames
479 result = out_q.get(True, 10)
480 #we will get an empty exception before this runs.
481 if not result:
482 threads_remaining -= 1
483 else:
484 #run() is a generator, and yields results from the work queue
485 yield result
486 except Exception as e:
487 #The cx_freeze version uses queue.Empty instead of Queue.Empty :(
488 if type(e) == Queue.Empty or str(type(e)) == "<class 'queue.Empty'>":
489 pass
490 else:
491 raise(e)
492 #make sure everyone is complete
493 if threads_remaining <= 0:
494 break
495 trace("killing nameserver process")
496 #We no longer require name servers.
497 try:
498 killproc(pid = verify_nameservers_proc.pid)
499 except:
500 #Windows threading.tread
501 verify_nameservers_proc.end()
502 trace("End")
503
504#exit handler for signals. So ctrl+c will work.
505#The 'multiprocessing' library each process is it's own process which side-steps the GIL
506#If the user wants to exit prematurely, each process must be killed.
507def killproc(signum = 0, frame = 0, pid = False):
508 if not pid:
509 pid = os.getpid()
510 if sys.platform.startswith('win'):
511 try:
512 kernel32 = ctypes.windll.kernel32
513 handle = kernel32.OpenProcess(1, 0, pid)
514 kernel32.TerminateProcess(handle, 0)
515 except:
516 #Oah windows.
517 pass
518 else:
519 os.kill(pid, 9)
520
521#Toggle debug output
522verbose = False
523def trace(*args, **kwargs):
524 if verbose or False:
525 for a in args:
526 sys.stderr.write(str(a))
527 sys.stderr.write(" ")
528 sys.stderr.write("\n")
529
530def error(*args, **kwargs):
531 for a in args:
532 sys.stderr.write(str(a))
533 sys.stderr.write(" ")
534 sys.stderr.write("\n")
535 sys.exit(1)
536
537def check_open(input_file):
538 ret = []
539 #If we can't find a resolver from an input file, then we need to improvise.
540 try:
541 ret = open(input_file).readlines()
542 except:
543 error("File not found:", input_file)
544 if not len(ret):
545 error("File is empty:", input_file)
546 return ret
547
548#Every 'multiprocessing' process needs a signal handler.
549#All processes need to die, we don't want to leave zombies.
550def signal_init():
551 #Escliate signal to prevent zombies.
552 signal.signal(signal.SIGINT, killproc)
553 try:
554 signal.signal(signal.SIGTSTP, killproc)
555 signal.signal(signal.SIGQUIT, killproc)
556 except:
557 #Windows
558 pass
559
560if __name__ == "__main__":
561 if getattr(sys, 'frozen', False):
562 # cx_freeze windows:
563 base_path = os.path.dirname(sys.executable)
564 multiprocessing.freeze_support()
565 else:
566 #everything else:
567 base_path = os.path.dirname(os.path.realpath(__file__))
568 parser = optparse.OptionParser("usage: %prog [options] target")
569 parser.add_option("-s", "--subs", dest = "subs", default = os.path.join(base_path, "names.txt"),
570 type = "string", help = "(optional) list of subdomains, default = 'names.txt'")
571 parser.add_option("-r", "--resolvers", dest = "resolvers", default = os.path.join(base_path, "resolvers.txt"),
572 type = "string", help = "(optional) A list of DNS resolvers, if this list is empty it will OS's internal resolver default = 'resolvers.txt'")
573 parser.add_option("-t", "--targets_file", dest = "targets", default = "",
574 type = "string", help = "(optional) A file containing a newline delimited list of domains to brute force.")
575 parser.add_option("-o", "--output", dest = "output", default = False, help = "(optional) Output to file (Greppable Format)")
576 parser.add_option("-j", "--json", dest="json", default = False, help="(optional) Output to file (JSON Format)")
577 parser.add_option("-a", "-A", action = 'store_true', dest = "ipv4", default = False,
578 help = "(optional) Print all IPv4 addresses for sub domains (default = off).")
579 parser.add_option("--type", dest = "type", default = False,
580 type = "string", help = "(optional) Print all reponses for an arbitrary DNS record type (CNAME, AAAA, TXT, SOA, MX...)")
581 parser.add_option("-c", "--process_count", dest = "process_count",
582 default = 16, type = "int",
583 help = "(optional) Number of lookup theads to run. default = 16")
584 parser.add_option("-f", "--filter_subs", dest = "filter", default = "",
585 type = "string", help = "(optional) A file containing unorganized domain names which will be filtered into a list of subdomains sorted by frequency. This was used to build names.txt.")
586 parser.add_option("-v", "--verbose", action = 'store_true', dest = "verbose", default = False,
587 help = "(optional) Print debug information.")
588 (options, args) = parser.parse_args()
589
590
591 verbose = options.verbose
592
593 if len(args) < 1 and options.filter == "" and options.targets == "":
594 parser.error("You must provie a target. Use -h for help.")
595
596 if options.filter != "":
597 #cleanup this file and print it out
598 for d in extract_subdomains(options.filter):
599 print(d)
600 sys.exit()
601
602 if options.targets != "":
603 targets = check_open(options.targets) #the domains
604 else:
605 targets = args #multiple arguments on the cli: ./subbrute.py google.com gmail.com yahoo.com if (len(resolver_list) / 16) < options.process_count:
606
607 output = False
608 if options.output:
609 try:
610 output = open(options.output, "w")
611 except:
612 error("Failed writing to file:", options.output)
613
614 json_output = False
615 if options.json:
616 try:
617 json_output = open(options.json, "w")
618 except:
619 error("Failed writing to file:", options.json)
620
621 record_type = False
622 if options.ipv4:
623 record_type="A"
624 if options.type:
625 record_type = str(options.type).upper()
626
627 threads = []
628 for target in targets:
629 target = target.strip()
630 if target:
631
632 #target => domain
633 #record_type =>
634 #options.subs => file the contain the subdomains list
635 #options.process_count => process count default = 16
636 #options.resolvers => the resolvers file
637 #options.output
638 #options.json
639 print target, record_type, options.subs, options.resolvers, options.process_count, output, json_output
640 print_target(target, record_type, options.subs, options.resolvers, options.process_count, output, json_output)