· 2 years ago · May 03, 2023, 08:20 PM
1#!/usr/bin/python
2# Copyright (C) 2010 Michael Ligh
3#
4# This program is free software: you can redistribute it and/or modify
5# it under the terms of the GNU General Public License as published by
6# the Free Software Foundation, either version 3 of the License, or
7# (at your option) any later version.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program. If not, see <http://www.gnu.org/licenses/>.
16#
17# [NOTES] -----------------------------------------------------------
18# 1) Tested on Linux (Ubuntu), Windows XP/7, and Mac OS X
19# 2) You must NOT use this script if the respective vendors begin to
20# prohibit doing so in the future. You must consult all relevant
21# acceptible usage policies.
22#--------------------------------------------------------------------
23import urllib, urllib2
24import sys
25import httplib
26import os
27import re
28import time
29import hashlib
30import urlparse
31from optparse import OptionParser
32
33try:
34 from sqlite3 import *
35except ImportError:
36 print "Cannot import sqlite3, the database function is disabled."
37
38# This should be stock with Python2.6 but not supplied in Python2.5
39try:
40 import simplejson
41except ImportError:
42 print 'You must install simplejson for VirusTotal, see http://www.undefined.org/python/'
43
44DBNAME = "virus.db"
45MAXWAIT = (60*10) # ten minutes
46
47# You must fill this in for VirusTotal (see http://www.virustotal.com/advanced.html)
48VTAPIKEY=''
49
50class Jotti:
51 def __init__(self, file):
52 self.file = file
53
54 f = open(self.file, "rb")
55 self.content = f.read()
56 f.close()
57
58 self.headers = {
59 'User-Agent' : 'Jotti Uploader 0.0.1',
60 'Accept' : '*/*',
61 }
62
63 self.cookie = ''
64 self.apc = ''
65
66 def parse_response(self, results):
67
68 detects = {}
69
70 while results.find('scannerid') != -1:
71 offset = results.find('scannerid')
72 results = results[offset+12:]
73 vendor = results[0:results.find('\"')]
74 if vendor == '':
75 continue
76 offset = results.find('virusname')
77 if offset == -1:
78 continue
79 results = results[offset+12:]
80 virus = results[0:results.find('\"')]
81 if virus == '':
82 continue
83 detects[vendor] = virus.replace('\\', '')
84
85 return detects
86
87 def get_detects(self, analysis_url):
88
89 detects = {}
90 tries = 0
91
92 print "Trying to get results for the next %d seconds..." % MAXWAIT
93
94 while tries < 10:
95
96 print "Try %d" % tries
97
98 netloc = urlparse.urlparse(analysis_url)[1]
99 path = urlparse.urlparse(analysis_url)[2]
100
101 try:
102 conn = httplib.HTTPConnection(netloc)
103 conn.request('GET', path, None, self.headers)
104 results = conn.getresponse().read()
105 except Exception, e:
106 print "Error parsing response: %s" % e
107 break
108
109 if results.find('scanid:') != -1:
110 scanid = results[results.find('scanid:')+9:]
111 scanid = scanid[0:scanid.find('\"')]
112
113 if scanid == '':
114 tries += 1
115 time.sleep(MAXWAIT/10)
116 continue
117
118 #print "Initialized scanid: %s" % scanid
119 results = None
120
121 try:
122 conn = httplib.HTTPConnection(netloc)
123 results_url = '/nestor/getscanprogress.php?' + self.cookie + '&lang=en&scanid=' + scanid
124 conn.request('GET', results_url)
125 results = conn.getresponse().read()
126 except Exception, e:
127 print "Error querying progress: %s" % e
128 break
129
130 if results != None:
131 detects = self.parse_response(results)
132 if (detects != None) and (len(detects) > 0):
133 break
134
135 tries += 1
136 time.sleep(MAXWAIT/10)
137
138 return detects
139
140 def submit(self):
141
142 analysis_url = self.search_by_hash()
143
144 if analysis_url == None:
145 analysis_url = self.upload_file()
146 if analysis_url != None:
147 print "You can find the new analysis here: %s" % analysis_url
148 return self.get_detects(analysis_url)
149 elif analysis_url != "nosubmit":
150 print "You can find the existing analysis here: %s" % analysis_url
151 return self.get_detects(analysis_url)
152
153 return {}
154
155 def search_by_hash(self):
156
157 if self.cookie == '' or self.apc == '':
158 self.get_params()
159
160 print "Initialized session cookie: %s" % self.cookie
161 print "Initialized APC: %s" % self.apc
162
163 md5 = hashlib.md5(self.content).hexdigest().upper()
164
165 print "Checking Jotti's databse for file with MD5: %s" % md5
166 response = ''
167
168 try:
169 conn = httplib.HTTPConnection('virusscan.jotti.org')
170 query_url = '/nestor/getfileforhash.php?' + self.cookie + "&hash=" + md5 + "&output=json"
171 conn.request('GET', query_url)
172 response = conn.getresponse().read()
173
174 if response.find('FILE_NOT_FOUND') != -1:
175 print "The file does not already exist on Jotti..."
176 return None
177
178 if response.find('HASH_INVALID') != -1:
179 print "The hash format is invalid..."
180 return None
181
182 if response.startswith('false'):
183 print "The file exists, but analysis is incomplete (try again later)..."
184 return "nosubmit"
185
186 response = response.replace('\"', '')
187 return 'http://virusscan.jotti.org/en/scanresult/' + response
188 except Exception, e:
189 print "Error searching for hash: %s" % e
190 pass
191
192 return None
193
194 def get_params(self):
195
196 try:
197 conn = httplib.HTTPConnection('virusscan.jotti.org')
198 conn.request('GET', '/en')
199 response = conn.getresponse()
200 headers = response.getheader('set-cookie')
201 if headers.find('sessionid') == 0:
202 self.cookie = headers[0:headers.find(';')]
203 body = response.read()
204 var = body.find('APC_UPLOAD_PROGRESS')
205 if var != -1:
206 value = body[var:].find("value=")
207 if value != -1:
208 body = body[var+value+7:]
209 self.apc = body[0:body.find('\"')]
210 except Exception, e:
211 print "Error getting parameters: %s" % e
212 return
213
214 def upload_file(self):
215
216 if self.cookie == '' or self.apc == '':
217 self.get_params()
218
219 print "Attempting to upload the sample, please wait..."
220
221 my_headers = self.headers
222 my_headers['Content-Type'] = 'multipart/form-data; boundary=---------------------------7da29022600d6'
223 my_headers['Cookie'] = "%s; lang=en" % self.cookie
224
225 body = "-----------------------------7da29022600d6\r\n"
226 body += "Content-Disposition: form-data; name=\"APC_UPLOAD_PROGRESS\"\r\n\r\n"
227 body += "%s\r\n" % self.apc
228 body += "-----------------------------7da29022600d6\r\n"
229 body += "Content-Disposition: form-data; name=\"MAX_FILE_SIZE\"\r\n\r\n"
230 body += "15728640\r\n"
231 body += "-----------------------------7da29022600d6\r\n"
232 body += "Content-Disposition: form-data; name=\"scanfile\"; "
233 body += "filename=\"%s\"\r\n" % os.path.basename(self.file)
234 body += "Content-Type: application/octet-stream\r\n"
235 body += "\r\n"
236 body += self.content
237 body += "\r\n"
238 body += "-----------------------------7da29022600d6--\r\n"
239
240 try:
241 conn = httplib.HTTPConnection('virusscan.jotti.org')
242 conn.request('POST', '/processupload.php', body, my_headers)
243 response = conn.getresponse().read()
244 offset = response.find('top.location.href=')
245 #print response
246 if offset != -1:
247 response = response[offset+19:]
248 response = response[0:response.find('\"')]
249 return 'http://virusscan.jotti.org' + response
250 except Exception, e:
251 print "Error uploading file: %s" % e
252 pass
253
254 return None
255
256class ThreatExpert:
257 def __init__(self, file=None, md5=None):
258
259 self.md5 = md5
260
261 if file != None:
262 data = open(file, "rb").read()
263 self.md5 = hashlib.md5(data).hexdigest().lower()
264
265 def get_data_between(self, data, start_tag, end_tag):
266 start = data.find(start_tag)
267 if start != -1:
268 start += len(start_tag)
269 end = data[start:].find(end_tag)
270 if end != -1:
271 return data[start:start+end]
272 return None
273
274 def remove_html_tags(self, data):
275 p = re.compile(r'<.*?>')
276 return p.sub('', data)
277
278 def split_record(self, text):
279 parts = text.split(" ", 1)
280 vendor = parts[1].replace("[", "")
281 vendor = vendor.replace("]", "")
282 detect = parts[0]
283 return (vendor, detect)
284
285 def parse_response(self, response):
286
287 detects = {}
288
289 # handle multiple aliases (viruses only)
290 buf1 = self.get_data_between(response, "<li>Alias:</li>", "</ul>")
291 # handle multiple aliases (viruses and packers)
292 buf2 = self.get_data_between(response, "<li>Alias & packer info:</li>", "</ul>")
293
294 if buf1 != None:
295 aliases = re.findall("<li>.+</li>", buf1)
296
297 for alias in aliases:
298 text = self.remove_html_tags(alias)
299 vendor, detect = self.split_record(text)
300 detects[vendor] = detect
301 elif buf2 != None:
302 aliases = re.findall("<li>.+</li>", buf2)
303
304 for alias in aliases:
305 text = self.remove_html_tags(alias)
306 vendor, detect = self.split_record(text)
307 detects[vendor] = detect
308 else:
309 # handle single aliases (only 1 av result)
310 if response.find("<li>Alias: ") != -1:
311 buf = response[response.find("<li>Alias: ")+11:]
312 if buf.find("</li>") != -1:
313 text = buf[:buf.find("</li>")]
314 vendor, detect = self.split_record(text)
315 detects[vendor] = detect
316
317 return detects
318
319 def search_by_hash(self):
320
321 search_url = 'http://www.threatexpert.com/report.aspx?md5=' + self.md5
322
323 print "Checking ThreatExpert for file with MD5: %s" % self.md5
324
325 try:
326 conn = httplib.HTTPConnection('www.threatexpert.com')
327 conn.request('GET', '/report.aspx?md5=' + self.md5)
328 response = conn.getresponse().read()
329 if response.find('Submission Summary') != -1:
330 print "Analysis exists: %s" % search_url
331 return response
332 else:
333 print "Analysis does not yet exist!"
334 except Exception, e:
335 print "Error searching for hash: %s" % e
336 pass
337
338 return
339
340 def submit(self):
341
342 detects = {}
343 response = self.search_by_hash()
344 if response != None:
345 detects = self.parse_response(response)
346
347 return detects
348
349class NoVirusThanks:
350 def __init__(self, file):
351 self.file = file
352
353 f = open(self.file, "rb")
354 self.content = f.read()
355 f.close()
356
357 self.headers = {
358 'User-Agent' : 'NoVirusThanks Uploader 0.0.1',
359 'Accept' : '*/*',
360 }
361
362 def get_data_between(self, data, start_tag, end_tag):
363 start = data.find(start_tag)
364 if start != -1:
365 start += len(start_tag)
366 end = data[start:].find(end_tag)
367 if end != -1:
368 return data[start:start+end]
369 return None
370
371 def remove_html_tags(self, data):
372 p = re.compile(r'<.*?>')
373 return p.sub(' ', data)
374
375 def parse_response(self, location):
376
377 detects = {}
378 location = location.replace("file", "analysis")
379 print location
380 tries = 0
381
382 while tries < 10:
383
384 print "Try %d" % tries
385
386 netloc = urlparse.urlparse(location)[1]
387 path = urlparse.urlparse(location)[2]
388
389 try:
390 conn = httplib.HTTPConnection(netloc)
391 conn.request('GET', path, None, self.headers)
392 results = conn.getresponse().read()
393 except Exception, e:
394 print "Error parsing response: %s" % e
395 break
396
397 if results.find("Error: No report found") != -1:
398 tries += 1
399 time.sleep(MAXWAIT/10)
400 continue
401
402 buf = self.get_data_between(results, '<!-- Virus information table -->', '</table>')
403 if buf != None:
404 buf = self.get_data_between(buf, '<tbody>', '</tbody>')
405 if buf != None:
406 rows = buf.split("<tr>")
407 for row in rows:
408 text = self.remove_html_tags(row)
409 cols = text.split(" ")
410 vendor = cols[0].strip()
411 if vendor == "":
412 continue
413 try:
414 detect = cols[3].strip()
415 except:
416 continue
417 if detect == "" or detect == "-":
418 continue
419 detects[vendor] = detect
420 break
421
422 return detects
423
424 def upload_file(self):
425
426 print "Submitting file to NoVirusThanks, please wait..."
427
428 my_headers = self.headers
429 my_headers['Content-Type'] = 'multipart/form-data; boundary=---------------------------47972514120'
430
431 body = "-----------------------------47972514120\r\n"
432 body += "Content-Disposition: form-data; name=\"upfile\"; "
433 body += "filename=\"%s\"\r\n" % os.path.basename(self.file)
434 body += "Content-Type: application/octet-stream\r\n"
435 body += "\r\n"
436 body += self.content
437 body += "\r\n"
438 body += "-----------------------------47972514120\r\n"
439 body += "Content-Disposition: form-data; name=\"submitfile\"\r\n"
440 body += "\r\n"
441 body += "Submit File\r\n"
442 body += "-----------------------------47972514120--\r\n"
443
444 try:
445 conn = httplib.HTTPConnection('vscan.novirusthanks.org')
446 conn.request('POST', '/', body, my_headers)
447 response = conn.getresponse()
448 location = response.getheader('location')
449 except Exception, e:
450 print "Error uploading file: %s" % e
451 return None
452
453 return location
454
455 def submit(self):
456 detects = {}
457 location = self.upload_file()
458 if location != None:
459 detects = self.parse_response(location)
460 return detects
461
462## {{{ http://code.activestate.com/recipes/146306/ (r1)
463import httplib, mimetypes
464
465def post_multipart(host, selector, fields, files):
466 """
467 Post fields and files to an http host as multipart/form-data.
468 fields is a sequence of (name, value) elements for regular form fields.
469 files is a sequence of (name, filename, value) elements for data to be uploaded as files
470 Return the server's response page.
471 """
472 content_type, body = encode_multipart_formdata(fields, files)
473 h = httplib.HTTP(host)
474 h.putrequest('POST', selector)
475 h.putheader('content-type', content_type)
476 h.putheader('content-length', str(len(body)))
477 h.endheaders()
478 h.send(body)
479 errcode, errmsg, headers = h.getreply()
480 return h.file.read()
481
482def encode_multipart_formdata(fields, files):
483 """
484 fields is a sequence of (name, value) elements for regular form fields.
485 files is a sequence of (name, filename, value) elements for data to be uploaded as files
486 Return (content_type, body) ready for httplib.HTTP instance
487 """
488 BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
489 CRLF = '\r\n'
490 L = []
491 for (key, value) in fields:
492 L.append('--' + BOUNDARY)
493 L.append('Content-Disposition: form-data; name="%s"' % key)
494 L.append('')
495 L.append(value)
496 for (key, filename, value) in files:
497 L.append('--' + BOUNDARY)
498 L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
499 L.append('Content-Type: %s' % get_content_type(filename))
500 L.append('')
501 L.append(value)
502 L.append('--' + BOUNDARY + '--')
503 L.append('')
504 body = CRLF.join(L)
505 content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
506 return content_type, body
507
508def get_content_type(filename):
509 return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
510## end of http://code.activestate.com/recipes/146306/ }}}
511
512class VirusTotal:
513 def __init__(self, file):
514 self.file = file
515
516 f = open(self.file, "rb")
517 self.content = f.read()
518 f.close()
519
520 def check(self, res):
521 url = "https://www.virustotal.com/api/get_file_report.json"
522 parameters = {"resource": res,
523 "key": VTAPIKEY}
524 data = urllib.urlencode(parameters)
525 req = urllib2.Request(url, data)
526 response = urllib2.urlopen(req)
527 json = response.read()
528 response_dict = simplejson.loads(json)
529 try:
530 return response_dict.get("report")[1]
531 except:
532 return {}
533
534 def upload_file(self):
535 host = "www.virustotal.com"
536 selector = "http://www.virustotal.com/api/scan_file.json"
537 fields = [("key", VTAPIKEY)]
538 file_to_send = self.content
539 files = [("file", os.path.basename(self.file), file_to_send)]
540 return post_multipart(host, selector, fields, files)
541
542 def submit(self):
543 resource = hashlib.md5(self.content).hexdigest()
544 detects = self.check(resource)
545 if len(detects) > 0:
546 print 'File already exists on VirusTotal!'
547 return detects
548 print 'File does not exist on VirusTotal, uploading...'
549 json = self.upload_file()
550 if json.find("scan_id") != -1:
551 offset = json.find("scan_id") + len("scan_id") + 4
552 scan_id = json[offset:]
553 scan_id = scan_id[:scan_id.find("\"")]
554 print 'Trying scan_id %s for %d seconds' % (scan_id, MAXWAIT)
555 i = 0
556 while i<(MAXWAIT/10):
557 detects = self.check(scan_id)
558 if len(detects) > 0:
559 return detects
560 time.sleep(MAXWAIT/10)
561 i += 1
562 return {}
563
564def savetodb(filename, detects, force):
565
566 if len(detects) == 0:
567 print "Nothing to add, submission failed."
568 return
569
570 if not os.path.isfile(DBNAME):
571 print "%s does not exist, try initialization first." % DBNAME
572 return
573
574 conn = connect(DBNAME)
575 curs = conn.cursor()
576
577 md5 = hashlib.md5(open(filename, 'rb').read()).hexdigest()
578
579 curs.execute("SELECT id FROM samples WHERE md5=?", (md5,))
580 ids = curs.fetchall()
581
582 if len(ids):
583 if not force:
584 ids = ["%d" % id[0] for id in ids]
585 print "The sample exists in virus.db with ID %s" % (','.join(ids))
586 print "Use the -o or --overwrite option to force"
587 return
588 else:
589 curs.execute("DELETE FROM samples WHERE md5=?", (md5,))
590
591 try:
592 curs.execute("INSERT INTO samples VALUES (NULL,?)", (md5,))
593 except Exception, e:
594 print "Error inserting record: %s" % e
595 print "Is your virus.db in a writable path?"
596 return
597
598 sid = curs.lastrowid
599 print "Added sample to database with ID %d" % sid
600 for key,val in detects.items():
601 curs.execute("INSERT INTO detects VALUES (NULL,?,?,?)", (sid, key, val))
602
603 conn.commit()
604 curs.close()
605
606def initdb():
607
608 if os.path.isfile(DBNAME):
609 print "File already exists, initialization not required."
610 return
611
612 conn = connect(DBNAME)
613 curs = conn.cursor()
614
615 curs.executescript("""
616 CREATE TABLE samples (
617 id INTEGER PRIMARY KEY,
618 md5 TEXT
619 );
620
621 CREATE TABLE detects (
622 id INTEGER PRIMARY KEY,
623 sid INTEGER,
624 vendor TEXT,
625 name TEXT
626 );
627 """)
628
629 curs.close()
630
631 if os.path.isfile(DBNAME):
632 print "Success."
633 else:
634 print "Failed."
635
636def main():
637 parser = OptionParser()
638 parser.add_option("-i", "--init", action="store_true",
639 dest="init", default=False, help="initialize virus.db")
640 parser.add_option("-o", "--overwrite", action="store_true",
641 dest="force", default=False,
642 help="overwrite existing DB entry")
643 parser.add_option("-f", "--file", action="store", dest="filename",
644 type="string", help="upload FILENAME")
645 parser.add_option("-v", "--virustotal", action="store_true",
646 dest="virustotal", default=False,
647 help="use VirusTotal")
648 parser.add_option("-e", "--threatexpert", action="store_true",
649 dest="threatexpert", default=False,
650 help="use ThreatExpert")
651 parser.add_option("-j", "--jotti", action="store_true",
652 dest="jotti", default=False,
653 help="use Jotti")
654 parser.add_option("-n", "--novirus", action="store_true",
655 dest="novirus", default=False,
656 help="use NoVirusThanks")
657
658 (opts, args) = parser.parse_args()
659
660 if opts.init:
661 initdb()
662 sys.exit()
663
664 if opts.filename == None:
665 parser.print_help()
666 parser.error("You must supply a filename!")
667 if not opts.virustotal and not opts.threatexpert and not opts.jotti and not opts.novirus:
668 parser.print_help()
669 parser.error("You must supply an action!")
670
671 if not os.path.isfile(opts.filename):
672 parser.error("%s does not exist" % opts.filename)
673
674 if opts.virustotal:
675 print "Using VirusTotal..."
676 if not sys.modules.has_key("simplejson"):
677 print 'You must install simplejson'
678 sys.exit()
679 vt = VirusTotal(opts.filename)
680 detects = vt.submit()
681 for key,val in detects.items():
682 print " %s => %s" % (key, val)
683 savetodb(opts.filename, detects, opts.force)
684 print
685
686 if opts.jotti:
687 print "Using Jotti..."
688 jotti = Jotti(opts.filename)
689 detects = jotti.submit()
690 for key,val in detects.items():
691 print " %s => %s" % (key, val)
692 savetodb(opts.filename, detects, opts.force)
693 print
694
695 if opts.threatexpert:
696 print "Using ThreatExpert..."
697 te = ThreatExpert(opts.filename)
698 detects = te.submit()
699 for key,val in detects.items():
700 print " %s => %s" % (key, val)
701 savetodb(opts.filename, detects, opts.force)
702 print
703
704 if opts.novirus:
705 print "Using NoVirusThanks..."
706 nvt = NoVirusThanks(opts.filename)
707 detects = nvt.submit()
708 for key,val in detects.items():
709 print " %s => %s" % (key, val)
710
711if __name__ == '__main__':
712 main()
713