gxBfHcji

1# from peewee import *
2from peewee import Model, CharField, IntegerField, FloatField, DateTimeField, ForeignKeyField, PrimaryKeyField
3from peewee import SqliteDatabase
4from peewee import OperationalError, IntegrityError
5
6from colorama import Fore,Style,Back
7from uuid import uuid1
8from sh import Command
9import pprint
10
11import copy
12import time
13
14import os,sys
15import hashlib, base64
16import argparse
17
18import itertools
19
20from web import sqlite_web
21
22"""
23    rule table, one by one
24"""
25
26ALLOWED = ('asp', 'aspx', 'jsp', 'jspx', 'php','ps1','psm1')
27DBPATH = os.path.abspath('hardsignture.db')
28DB = SqliteDatabase(DBPATH, autocommit=True, autorollback=True)
29SUMSCHEMA = {
30    "md5sum": '',
31    "sha1sum": '',
32    "filename": '',
33    "filepath":'',
34    "cmsname": '',
35    "cmstype": '',
36    "cmsversion": 0,        # 0 not check, temp here
37    "rulehited": ''         
38}
39
40RULESCANSCHEMA = {
41    "uid":'',
42    "rid": '',
43    "rname":'',
44    "rpath":'',
45    "rcontent": '',
46    "rtype": '',
47    'rtarget':'',
48    "rhit" : '',
49    "rscantime": time.asctime()
50}
51
52YARARULESCHEMA = {
53
54    "ruleId": '',
55    "ruleName":'',
56    "ruleDescription":'',
57    "ruleAuthor":'',
58    "ruleConditions":'',
59    "ruleScope":'',
60    "rulePrivilege":''
61}
62
63class YaraRuleScan(Model):
64    uid = CharField(primary_key=True)              # 
65    rid = CharField()                 # rulemd5
66    rcontent = CharField()            # base64 store
67    rname = CharField()
68    rpath = CharField()
69    # Include Rule Type, Common Whitelist BlackList  'B' 'C' 'W'  Avoid come same time
70    rtype = CharField()
71    rtarget = CharField()
72    rhit = IntegerField()
73    rscantime = DateTimeField()
74
75    class Meta:
76        database = DB
77
78class YaraRule(Model):
79    """
80        后期在考虑要不要添加具体规则进去,当然是极好的.
81    """
82    ruleId = CharField(primary_key=True)
83    ruleName = CharField()
84    ruleDescription = CharField()
85    ruleAuthor = CharField()
86    ruleConditions = CharField()
87    ruleScope = CharField()             #"Global or not"
88    rulePrivilege = CharField()         #"Private or not"
89
90    class Meta:
91        database = DB
92
93class HardSum(Model):
94    # uid = CharField(primary_key=True)
95    sha1sum = CharField(primary_key=True)          # primary_key=True
96    md5sum = CharField()
97    rulehited = CharField()  # ForeignKeyField(YaraRuleScan, related_name='relationships')
98
99    filename = CharField()
100    filepath = CharField()
101    cmsname = CharField()
102    cmstype = CharField()
103    cmsversion = FloatField()
104
105    class Meta:
106        database = DB         
107
108
109def cleanlistofdict(listofdict, key):
110    """
111        [{},{},{}...{},{}] remove duplicate dict by keywords
112    """
113
114    try:
115        cleanunique = list({v[key]: v for v in listofdict}.values())
116    except Exception as e:
117        pass
118    else:
119        return cleanunique
120
121def difflistofdict(data1, data2):
122    """
123        return list of diff
124    """
125    diff = list(itertools.filterfalse(lambda x: x in data1, data2)) + \
126      list(itertools.filterfalse(lambda x: x in data2, data1))
127    return diff
128
129def allowedsuffix(filepath):
130    suffix = [x for x in os.path.basename(filepath).split('.') if x in ALLOWED]
131    if suffix:
132        return True
133
134def md5sumfile(file_full_path):
135    if(os.path.isfile(file_full_path)):
136        return hashlib.md5(open(file_full_path, 'rb').read()).hexdigest()
137    else:
138        return None
139
140def sha1sumfile(file_full_path):
141    if(os.path.isfile(file_full_path)):
142        return hashlib.sha1(open(file_full_path, 'rb').read()).hexdigest()
143    else:
144        return None
145
146
147def walkdir(dir_path):
148    """
149        此处去重用于减少扫描速度.
150    """
151    filepath = []
152    for rt, dirs, files in os.walk(dir_path):
153        for f in files:
154            if allowedsuffix(f.lower()):
155            # if f.lower().endswith(ALLOWED):
156                # filepath.append()
157
158                
159                _file_path = os.path.join(rt, f)
160                tmp = {"md5":md5sumfile(_file_path),'filepath':_file_path}
161
162                filepath.append(copy.deepcopy(tmp))
163
164    res = cleanlistofdict(filepath,"md5")
165    filepaths = [x["filepath"] for x in res]
166
167    return filepaths
168
169def scan_dir(dir_path, rules_path='', rules_type=''):
170
171    """
172    """
173
174    import yara
175
176    checkresults = []
177    skip = False
178
179    filepropertydict = copy.deepcopy(SUMSCHEMA)
180    ruleproperty = copy.deepcopy(RULESCANSCHEMA)
181
182    def compilerules(rules_path):
183        if os.path.isfile(rules_path):
184            try:
185                comprule = yara.load(rules_path)
186            except Exception as e:
187
188                print(Fore.RED + "This Is Not A Compiled Yara Rule, Now we will try to compiled it"+ Fore.RESET)
189                try:
190                    comprule = yara.compile(rules_path)
191                except Exception as e:
192
193                    print(Fore.RED + "Compiled Error, Yara-python lib was not able to compiled netseted rule. Or Your rule was wrong." + Fore.RESET)
194                    print(Fore.RED + "Now, Programer will exit." + Fore.RESET)
195
196                    sys.exit(1)
197
198                else:
199                    return comprule
200            else:
201                return comprule
202
203    if rules_path:
204        processedrules = compilerules(rules_path)
205
206        # with open(rules_path, 'rb') as ff:
207        #     rcontent = base64.b64encode(ff.read())
208        # ruleproperty['rcontent'] = rcontent
209
210        ruleproperty['uid'] = str(uuid1())
211        ruleproperty['rid'] = sha1sumfile(rules_path)
212        ruleproperty['rname'] = os.path.basename(rules_path)
213        ruleproperty['rpath'] = os.path.abspath(rules_path)
214        ruleproperty['rtarget'] = dir_path
215        ruleproperty['rtype'] = rules_type
216
217        skip = True
218
219    if os.path.exists(dir_path) and os.path.isdir(dir_path):
220        if skip:
221            print("Now We Would Scan Dir With PreCompiled Rules.\n")
222            print("Session Id:\n\t" + Fore.GREEN + ruleproperty['uid'])
223            print(Fore.YELLOW + "Please Remeber This Session Id, You can use it to export the file hash" + Fore.RESET)
224
225            for file_path in walkdir(dir_path):
226                # Compute Yara Rule Hit, if Hit Store to database
227                with open(file_path, 'rb') as readyfile:
228                    matches = processedrules.match(data=readyfile.read())
229                    if matches is not None :
230                        filepropertydict['rulehited'] = ruleproperty['uid']
231                        filepropertydict['md5sum'] = md5sumfile(file_path)
232                        filepropertydict['sha1sum'] = sha1sumfile(file_path)
233                        filepropertydict['filepath'] = file_path
234                        filepropertydict['filename'] = os.path.basename(file_path)
235                        
236                        # cms 识别怎么做
237                        # filepropertydict['cmsname'] =
238                        # filepropertydict['cmstype'] =
239                        # filepropertydict['cmsversion'] =
240
241                        checkresults.append(copy.deepcopy(filepropertydict))
242
243            checkres = cleanlistofdict(checkresults, 'sha1sum')
244            ruleproperty['rhit'] = len(checkres)
245
246
247
248        else:
249            # Scan All
250            for file_path in walkdir(dir_path):
251                
252                filepropertydict['md5sum'] = md5sumfile(file_path)
253                filepropertydict['sha1sum'] = sha1sumfile(file_path)
254                filepropertydict['filename'] = os.path.basename(file_path)
255
256                checkresults.append(copy.deepcopy(filepropertydict))
257
258            checkres = cleanlistofdict(checkresults, 'sha1sum')
259
260        return checkres, ruleproperty
261    else:
262        print(Fore.RED + " Your path is not a directory, Please select a correctly directory \n" + Fore.RESET)
263        return None
264    print(Fore.BLUE + "Check Directory Was Done\n" + Fore.RESET)
265
266def shellcompileRule(yarabin,rule,output):
267    """
268        compile rule as one file for next scan, this way can compiled the netsted rule in one file.
269    """
270    if os.path.isfile(yarabin) and os.path.isfile(rule) and os.path.isfile(output):
271        compilerulebysh = "{0} -d detect=signature {1} {2}".format(yarabin, rule, output)
272
273        try:
274            Command(compilerulebysh)
275
276        except Exception as e:
277            print(Fore.RED + "Compiled Rules From Command Was Wrong" + Fore.RESET)
278
279    else:
280        print(Fore.RED + "Your Path Was Not File")
281
282def writerestofile(checkresults, filename, rulename):
283    
284    a = "import \"hash\" \n"
285    b = "private rule {} \n".format(rulename)
286    c = "{\n"
287    d = "    condition:\n"
288
289    header = a + b + c + d
290
291    with open(filename, 'a') as ww:
292        ww.write(header)
293        for _ in checkresults:
294            if checkresults[-1] == _:
295                ww.write("\t\t\t\thash.sha1(0,filesize) == \"{}\"".format(_))
296            else:
297                ww.write("\t\t\t\thash.sha1(0,filesize) == \"{}\"".format(_))
298                ww.write('\tor\n')
299        ww.write('\n}')
300
301def removesession(sessionid):   
302
303    print("Now We will delete sessionID :\t\t" +
304          Fore.RED+ sessionid + Fore.RESET)
305
306    with DB.transaction():
307        YaraRuleScan.delete().where(YaraRuleScan.uid == sessionid).execute()
308        HardSum.delete().where(HardSum.rulehited == sessionid).execute()
309
310def scanseesion2rulefile(filename, sessionid, rulename):
311    """
312        Read sha1sum list from db by ruleid and write it to a file  
313    """
314
315    if os.path.exists(filename):
316        os.remove(filename)
317
318    res = []
319
320    for item in HardSum.select().where(HardSum.rulehited == sessionid):
321        res.append(item.sha1sum)
322    writerestofile(res, filename, rulename)
323
324def rulefile2scanseesion(rulefile, ruletype, sessionid=None,rhit=-1):
325
326    if sessionid == None:
327        sessionid = str(uuid1())
328        print(Fore.YELLOW + "You didn't passed a sessionid,Now we will gererate random one." + Fore.RESET)
329        print(Fore.GREEN + "SessionID:\n\t" + Fore.WHITE + sessionid + Fore.RESET )
330
331    checkres = []
332    ruleproperty = copy.deepcopy(SUMSCHEMA)
333    rulefileproperty = copy.deepcopy(RULESCANSCHEMA)
334
335    rulefileproperty['uid'] = sessionid
336    rulefileproperty['rid'] = sha1sumfile(rulefile)
337    rulefileproperty['rtype'] = ruletype
338    rulefileproperty['rname'] = os.path.basename(rulefile)
339    rulefileproperty['rpath'] = os.path.abspath(rulefile)
340    rulefileproperty['rhit'] = rhit
341
342    if os.path.isfile(rulefile):
343        with open(rulefile) as f:
344            for line in f.readlines():
345                tmp = line.strip("\t ")     #不要忘了strip空格
346                if tmp.startswith("hash"):
347                    ruleproperty['rulehited'] = sessionid
348                    hashvalue = tmp.split("\"")[1]
349
350                    # if len(hashvalue) == 40:
351                    ruleproperty['sha1sum'] = hashvalue  
352                    # else:
353                        # ruleproperty['md5sum'] = hashvalue
354                    
355                    checkres.append(copy.deepcopy(ruleproperty))
356
357        return cleanlistofdict(checkres,"sha1sum"), rulefileproperty
358    else:
359        print(Fore.RED + "Make Sure Your File Exist And It IS A Valid Rule File" + Fore.RESET)
360        sys.exit(250)
361
362
363def initDB():
364    """
365        初始化表
366
367        TODO:
368            SCHEMA和已知表的比对,如果现有数据库内的表结构和预定义SCHEMA不同,DROP当前表
369    """
370    try:
371        DB.connect()
372        if not HardSum.table_exists():
373            print(Fore.YELLOW + "HardSum Table Not Exist, And Now We Will Create It" + Fore.RESET)
374            DB.create_table(HardSum)
375
376        if not YaraRuleScan.table_exists():
377            print(Fore.YELLOW + "YaraRuleScan Table Not Exist, And Now We Will Create It" + Fore.RESET)
378            DB.create_table(YaraRuleScan)
379
380        if not YaraRule.table_exists():
381            print(Fore.YELLOW + "YaraRule Table Not Exist, And Now We Will Create It" + Fore.RESET)
382            DB.create_table(YaraRule)
383
384    except Exception as e:
385        print(e)
386
387
388def insertDB(checkres,ruleproperty,step=100):
389    """ 
390        默认以100为单位,按批插入,每次插入前,会检查是否在已知数据库中.目前还是采用了通过先检测的机制,当然,依旧可以按1步长插入.
391        ruletype: blacklists whitlists 
392    """
393    # checkres, ruleproperty = scan_dir(rules_path=rule_path, dir_path=dir_path, rules_type=ruletype)
394    
395    already = []
396
397    for item in HardSum.select():
398        already.append(item.sha1sum)
399        # already.append(item.md5sum)
400
401    checkres = [item for item in checkres if item['sha1sum'] not in already]
402    # checkres = [item for item in checkres if item['md5sum'] not in already]
403    
404
405    if checkres:
406        with DB.atomic():
407            
408            for idx in range(0, len(checkres), step):
409                try:
410                    HardSum.insert_many(checkres[idx:idx + step]).execute()
411
412            # if YaraRuleScan.select().where(ruleproperty['rid']):
413            #         YaraRuleScan.update(ruleproperty).where(
414            #             YaraRuleScan.rid == ruleproperty['rid']).execute()
415            # else:
416                except OperationalError as e:
417                    print(Fore.RED + " Database is locked" + Fore.RESET)
418                    pass
419                except IntegrityError as e:
420                    print(Fore.RED + "\n\nNotices:\" This Item Exists, It's will not be Changed\" " + Fore.RESET)
421                    pass
422
423            YaraRuleScan.create(**ruleproperty)
424    else:
425        print(Fore.YELLOW + "May Be Your Rule Data is Null, May Be it's already in DB" + Fore.RESET)
426
427
428def exportDB(ruletype):
429    """
430            先去YaraRuleScan里读取某种规则类型,根据ruleid查找sessionid.然后导出所有sessionid下的数据,去重即可.
431    """
432    pass
433
434def readDB():
435    pass
436
437def listDB(limit):
438    
439    for item in YaraRuleScan.select().order_by(YaraRuleScan.rscantime.desc()).limit(limit):
440        print(Fore.GREEN + "sessionID:" + Fore.RESET, item.uid)
441        print(Fore.BLUE + "\truleID      " + Fore.RESET, item.rid)
442        print(Fore.BLUE + "\truleHit     " + Fore.RESET, item.rhit)
443        print(Fore.BLUE + "\ttargetDir   " + Fore.RESET, item.rtarget)
444        print(Fore.BLUE + "\truleType    " + Fore.RESET, item.rtype)
445        print(Fore.BLUE + "\truleScanTime" + Fore.RESET, item.rscantime)
446
447def cleanDB():
448    """
449        将没有hardsum对应的YaraRuleScan删除
450    """
451    for rulefile in YaraRuleScan.select():
452        count = HardSum.select().where(HardSum.rulehited==rulefile.uid).count()
453        if count == 0:
454            print(Fore.YELLOW + "{} ".format(rulefile.uid) + Fore.RED + "Was Hit NULL, And We will remove it" + Fore.RESET)
455            YaraRuleScan.delete().where(YaraRuleScan.uid == rulefile.uid).execute()
456            # HardSum.delete().where(YaraRuleScan.rulehited == rulefile.uid)
457            print(Fore.GREEN + "{} remove Successful".format(rulefile.uid) +
458                  Fore.RESET)
459
460
461
462def getDiff(target):
463    """
464        得到最近两次对同一文件夹扫描之后的差异比较
465    """
466    queryres = YaraRuleScan.select().where(YaraRuleScan.rtarget==target).order_by(YaraRuleScan.rscantime.desc()).limit(2)
467    
468    if len(queryres) == 2:
469        res1 = HardSum.select().where(
470            HardSum.rulehited == queryres[0].uid)  # new
471        res2 = HardSum.select().where(
472            HardSum.rulehited == queryres[1].uid)  # old
473
474        data1 = [x.sha1sum for x in res1]
475        data2 = [y.sha1sum for y in res2]
476
477        somethingadd =    [item for item in data1 if item not in data2]
478        somethingremove = [item for item in data2 if item not in data1]
479        print("This data is Only For Static Analysis, It won't be changed with your database" + Fore.YELLOW + "\n{}:".format(target))
480        print(Fore.GREEN + "\tAdded: {}".format(len(somethingadd)) + Fore.RESET)
481        print(Fore.RED + "\tRemoved: {}".format(len(somethingremove)) + Fore.RESET)
482
483    else:
484        print(Fore.RED + "You must make sure your directory was scaned by twice and store in the db" + Fore.RESET)
485
486if __name__ == '__main__':
487
488    parser = argparse.ArgumentParser(description='checksum directory via the command line')
489
490    parser.add_argument('-a', '--all',      help='Check sum full dir without rules scan ',action='store_true')
491    parser.add_argument('-c', '--compare',  help='Compare latest two scan in your target ',action='store')
492    
493    parser.add_argument('-d', '--dirname',  help='Target directory',action='store')
494    parser.add_argument('-x', '--GUI',  help='Web GUI for sqlite database', action='store_true')
495    parser.add_argument('-l', '--listsession', nargs='?',help='Default list last session, you can change it ', action='store', type=int, const=1)
496    
497    parser.add_argument('-i', '--input', help='input file ', action='store')    
498    parser.add_argument('-o', '--output',help='output file', action='store')
499
500    parser.add_argument('-n', '--rulename', help='Yara Rule Name',action='store')
501
502    parser.add_argument('-r', '--rulefile', help='Read Rule Scan',action='store')
503    parser.add_argument('-s', '--session',  help='Outputfile By Session id',action='store')
504    parser.add_argument('-t', '--ruletype', help='Whitelist or Blacklist',action='store')
505
506    args = vars(parser.parse_args())
507
508    initDB()
509    cleanDB()
510
511    if args['GUI']:
512        sqlite_web.webgui(DBPATH)
513
514    elif args['compare']:
515        getDiff(args['compare'])
516
517    elif args['input'] and args['ruletype']:
518        if args['session']:
519            ires,irule  = rulefile2scanseesion(args['input'],args['ruletype'],args['session'],1)
520        else:
521            ires, irule = rulefile2scanseesion(args['input'],args['ruletype'],1)
522        insertDB(ires,irule)
523    
524    #Export Session Data from db to file
525    elif args['output'] and args['session'] and args['rulename']:
526        scanseesion2rulefile(args['output'], args['session'], args['rulename'])
527
528    # Scan All Dir Without Insert DB
529    # 此处考虑到白名单文件过多,所以选择全部扫描的,并不添加到数据库中,避免增大数据库.
530    elif args['dirname'] and args['output'] and args['all'] and args['rulename']:
531        res = scan_dir(args['dirname'])
532        writerestofile([o['sha1sum'] for o in res],args['output'],args['rulename'])
533
534    # Scan Dir filter by rules and should be insert to DB
535    elif args['dirname'] and args['rulefile'] and args['ruletype']:
536        res,ruleproperty = scan_dir(dir_path=args['dirname'], rules_path=args['rulefile'],rules_type=args['ruletype'])
537        insertDB(res,ruleproperty)
538        if args['output'] and args['rulename']:
539            scanseesion2rulefile(args['output'],ruleproperty['uid'], args['rulename'])
540    elif args['listsession']:
541        listDB(args['listsession'])
542    else:
543        parser.print_help()
544        print("Example:")
545        print("python hardrules.py -o ./some.yar -s 08e41284-f1d3-11e7-add6-6045cb8aeb58 -n wordpress")
546        print("python hardrules.py -i php-malware-finder/whitelists/DiscuzX.yar -t whitelists")
547        print("python hardrules.py -d ~/working/data/phpmyadmin -r ~/resoures/php-malware-finder/php-malware-finder/ceshi -t whitelists")
548        print("python hardrules.py -d ~/resoures/webshell/php/ -o ./some.yar -n ceshiyixa -t blacklists -r ~/resoures/php-malware-finder/php-malware-finder/ceshi")