· 4 years ago · Jun 29, 2021, 08:56 PM
1import click
2from importlib import import_module
3from Output import Output
4from Filter import Filter
5from datetime import datetime
6from Logging import log_setup
7from concurrent.futures import ProcessPoolExecutor
8from Shodan import Shodan
9import nmap3
10from pathlib import Path
11
12
13class NotRequiredIf(click.Option):
14 def __init__(self, *args, **kwargs):
15 self.not_required_if = kwargs.pop('not_required_if')
16 assert self.not_required_if, "'not_required_if' parameter required"
17 kwargs['help'] = (kwargs.get('help', '') +
18 ' NOTE: This argument is mutually exclusive with %s' %
19 self.not_required_if
20 ).strip()
21 super(NotRequiredIf, self).__init__(*args, **kwargs)
22
23 def handle_parse_result(self, ctx, opts, args):
24 other_present = self.not_required_if in opts
25
26 if other_present:
27 we_are_present = self.name in opts
28 if we_are_present:
29 raise click.UsageError(
30 "Illegal usage: `%s` is mutually exclusive with `%s`" % (
31 self.name, self.not_required_if))
32 else:
33 self.prompt = None
34
35 return super(NotRequiredIf, self).handle_parse_result(
36 ctx, opts, args)
37
38
39@click.command()
40@click.option("--hosts-file", "-h", help="Path to filename with hosts [IP:PORT] format, separated by a newline.",
41 cls=NotRequiredIf, not_required_if="shodan_stream")
42@click.option("--shodan-stream", "-ss", help="Use Shodan stream API to get hosts instead from hosts file.",
43 cls=NotRequiredIf, not_required_if="hosts_file")
44@click.option("--version-scan", "-v", is_flag=True, help="The program filters hosts to their suitable modules "
45 "(e.g., MongoDB, Cassandra, Elasticsearch) by comparing the "
46 "module/service's default port to the host's port."
47 " However, sometimes there are hosts with a port different from"
48 " the default one (e.g., port 9201 instead of 9200 for ES),"
49 " and in those cases, the program can use NMAP service scan to"
50 " find out the host's service product, thus matching it to its"
51 " suitable module.")
52@click.option("--patterns", "-p", help="Filter clusters by regex patterns."
53 " Path to regex patterns file, each pattern separated by a newline.")
54@click.option("--match-against", "-m", help="Where to match regex patterns.", type=click.Choice(
55 ["Databases names", "Documents names", "All"], case_sensitive=False), default="All")
56@click.option("--size", "-s", help="Filter clusters by size (in bytes)."
57 " For example, to filter clusters bigger than 10MB you would pass: '{\"bigger\": "
58 "10000000}'. To filter clusters bigger than 10MB but smaller than 100MB you would "
59 "pass: '{\"bigger\": 10000000, \"smaller\": 100000000}'.", type=str)
60@click.option("--output", "-o", is_flag=True, help="Output to file.")
61@click.option("--format", "-f", "format_", help="Output file name format.", type=click.Choice(
62 ["JSONLINES", "CSV", "TXT"], case_sensitive=False), default="TXT")
63@click.option("--exclude-unmatched", "-eu", is_flag=True, help="Exclude non-matching clusters in output.")
64@click.option("--include-geo", "-ig", is_flag=True, help="Include the IP country in output.")
65@click.option("--processes", help="Number of processes. Default 1", type=int, default=1)
66@click.option("--try-default", "-t", is_flag=True, help="If authentication to the cluster fail, try to login with "
67 "default credentials.")
68@click.option("--shodan-vulns", "-sv", is_flag=True, help="Get vulnerabilities of matched clusters using Shodan."
69 "Shodan API key is required. Input the API key at "
70 "config.config file.")
71@click.option("--silent", is_flag=True, help="No terminal output.")
72def main(hosts_file, shodan_stream, version_scan, patterns, match_against, size, output, format_,
73 exclude_unmatched, include_geo, processes, try_default, shodan_vulns, silent):
74 leak_finder = LeakFinder(hosts_file, shodan_stream, version_scan, patterns, match_against, size, output, format_,
75 exclude_unmatched, include_geo, processes, try_default, shodan_vulns, silent)
76 leak_finder.wrapper()
77
78
79class LeakFinder:
80 log = log_setup("LeakFinder")
81 cluster_ip = {"3306": "MySQL", "27017": "MongoDB", "9200": "ElasticSearch", "9042": "Cassandra"}
82 filename = datetime.now().strftime("%m.%d.%Y %H:%M:%S")
83 nmap = nmap3.Nmap()
84
85 def __init__(self, hosts_file, shodan_stream, version_scan, patterns, match_against, size, output, format_,
86 exclude_unmatched, include_geo, processes, try_default, shodan_vulns, silent):
87 self.hosts_file = hosts_file
88 self.shodan_stream = shodan_stream
89 self.version_scan = version_scan
90 self.patterns = patterns
91 self.match_against = match_against
92 self.size = size
93 self.output = output
94 self.format_ = format_
95 self.exclude_unmatched = exclude_unmatched
96 self.include_geo = include_geo
97 self.processes = processes
98 self.try_default = try_default
99 self.shodan_vulns = shodan_vulns
100 self.silent = silent
101
102 self.host, self.port, self.module_name, self.cluster_obj, self.cluster_instance = "", 0, "", None, None
103 self.filter_obj, self.info = None, None
104
105 def yield_hosts(self):
106 for line in self.read_gen():
107 line = line.strip().split(":")
108 host, port = line[0], int(line[1])
109 yield host, port
110
111 def main_method(self, connection_tuple):
112 self.host, self.port = connection_tuple
113 self.cluster_obj, self.module_name = self.get_cluster_object()
114 self.cluster_instance = self.cluster_obj(self.host, self.port, self.try_default)
115 print("self.cluster_instance:", self.cluster_instance)
116 if not self.cluster_instance.error:
117 print("ERROR:", self.cluster_instance.error)
118 self.cluster_method_manager()
119 self.filter_obj = Filter(self.cluster_instance, self.patterns, self.match_against, self.size)
120 if not self.exclude_unmatched or any(
121 (self.filter_obj.pattern_match, self.filter_obj.size_match)
122 ):
123 print("READY TO initialize Output object")
124 Output(self.info_builder(), f"OUTPUT {LeakFinder.filename}", self.output, self.format_,
125 self.exclude_unmatched, self.include_geo, self.silent)
126 print("Initialized Output object")
127
128 def wrapper(self):
129 self.valid_file(self.patterns, "patterns")
130 self.valid_file(self.hosts_file, "hosts_file")
131
132 executor = ProcessPoolExecutor(self.processes)
133
134 executor.map(self.main_method, [connection_tuple for connection_tuple in self.yield_hosts()])
135
136 @staticmethod
137 def valid_file(filepath, option): # sourcery skip: merge-nested-ifs
138 if filepath:
139 if not Path(filepath).is_file():
140 LeakFinder.log.critical(f"INVALID FILE PATH --{option}.")
141 exit()
142
143 def read_gen(self):
144 with open(self.hosts_file, "r") as f:
145 yield from f
146
147 def get_cluster_object(self):
148 self.module_name = LeakFinder.cluster_ip.get(str(self.port))
149 if not self.module_name:
150 if self.version_scan:
151 LeakFinder.log.warning(
152 f"Port {self.port} is not linked with any module. Trying NMAP service scan...")
153 self.module_name = [module_name for module_name in LeakFinder.cluster_ip.values() if
154 module_name.lower() in self.get_service().lower()]
155 if self.module_name:
156 self.module_name = self.module_name[-1]
157 else:
158 LeakFinder.log.warning(
159 f"Port {self.port} is not linked with any module. Available modules by host ports "
160 f"\n{LeakFinder.cluster_ip} "
161 )
162 else:
163 LeakFinder.log.warning(
164 f"Port {self.port} is not linked with any module. Available modules by host ports "
165 f"\n{LeakFinder.cluster_ip}")
166 return getattr(import_module(f"API.{self.module_name}"), self.module_name), self.module_name
167
168 def get_service(self):
169 try:
170 results = LeakFinder.nmap.scan_top_ports(self.host, args=f"-sV -Pn -p {self.port}")
171 return results.get(self.host).get("ports")[0].get("service").get("product")
172 except Exception:
173 pass
174
175 def info_builder(self):
176 # Returns a dictionary for class Output
177 print("info:", {"host": self.host, "port": self.port, "cluster_size": self.cluster_obj.cluster_size,
178 "module": self.module_name})
179 info = {"host": self.host, "port": self.port, "cluster_size": self.cluster_obj.cluster_size,
180 "module": self.module_name}
181 print("info:", info)
182 if self.filter_obj.pattern_match:
183 info["matches"] = self.filter_obj.matches
184 info["matched_against"] = self.return_matched_against({"regex_match": self.filter_obj.pattern_match,
185 "size_match": self.filter_obj.size_match})
186 if self.module_name in ("Cassandra", "MySQL") and self.cluster_obj.login_credentials:
187 info["login_credentials"] = str(self.cluster_obj.login_credentials).replace("{", "").replace("}",
188 "")
189 s = Shodan(self.host)
190 if (
191 any((self.filter_obj.size_match, self.filter_obj.pattern_match))
192 and not s.error
193 and not s.cancel
194 ) and self.shodan_vulns:
195 info["vulnerabilities"] = s.get_vulns()
196 print("info:", info)
197 return info
198
199 @staticmethod
200 def return_matched_against(matched_against_dict):
201 return [key.replace("_", " ").title() for key, value in matched_against_dict.items() if value]
202
203 def cluster_method_manager(self):
204 if self.patterns:
205 if self.match_against == "Databases names":
206 self.cluster_instance.list_database_names()
207 elif self.match_against == "Documents names":
208 self.cluster_instance.list_collections_names()
209 else:
210 self.cluster_instance.list_database_names()
211 self.cluster_instance.list_collections_names()
212 self.cluster_instance.get_total_size()
213
214
215if __name__ == "__main__":
216 main()
217