· 6 years ago · May 21, 2019, 11:18 AM
1# =============================================================================
2'''
3 File name: crqAutoDeploy.py
4 Description: Autodeployment Script
5 Author: Divit Sharma
6 Date created: 05/01/2019
7 Date last modified: 05/01/2019
8 Python Version: 3.7.1 ()
9 Execution : python3 crqAutoDeploy.py /path/crqAutoDeploy/crqAutoDeploy.conf
10'''
11# =============================================================================
12# Imports
13# =============================================================================
14
15import sys,os,logging,subprocess,time
16from datetime import date
17from datetime import datetime
18
19# =============================================================================
20# Code
21# =============================================================================
22
23
24#Logging
25logpath = "/path/crqAutoDeploy/logs"
26timeLog = str(datetime.now().strftime("%Y-%m-%d_%H_%M_%S"))
27print("Script Started at " + timeLog)
28
29logName = '{logpath}/crqAutoDeploy_{timeLog}.log'.format(timeLog = timeLog, logpath = logpath)
30try:
31 os.popen("ls {}".format(logpath.strip())).read()
32except:
33 sys.exit("Logpath not set")
34
35uid = os.popen("whoami").read().rstrip("\n")
36
37level = logging.INFO
38handlers = [logging.FileHandler(logName), logging.StreamHandler()]
39
40logging.basicConfig(handlers = handlers, format="%(levelname)s - {uid} - %(asctime)s - %(message)s".format(uid=uid),level = level,datefmt='%Y-%m-%d %H:%M:%S')
41
42# Check confFile exists or not
43if len(sys.argv) == 2:
44
45 confFile=sys.argv[1]
46
47 logging.info("confFile : " + confFile)
48
49else:
50 logging.error("Please check the arguments. Script should have 1 argument i.e. Absolute path of config file")
51 logging.error("Script Ended with Error")
52 sys.exit("Please check the arguments. Script should have 1 argument i.e. Absolute path of config file.\nScript Ended with Error")
53
54try:
55 os.popen("ls {}".format(confFile.strip())).read()
56except:
57 logging.error("Script ended with error. confFile not found at {}".format(confFile))
58 logging.error("Script ended with error")
59 sys.exit("Script ended with error. confFile not found at {}".format(confFile))
60
61logging.info("confFile found")
62logging.info("confFile should have entries in this format - key>value")
63
64# Read properties from confFile
65
66data = dict()
67
68with open(confFile, 'r') as raw_data:
69 for item in raw_data:
70 item = item.rstrip("\n").strip()
71 if item:
72 if '>' in item:
73 key,value = item.split('>', 1)
74 data[key]=value.strip()
75 else:
76 logging.error("Unknown Key - " + item)
77 logging.error("Script Ended with Error. Please check key file present at " + confFile)
78 sys.exit("Script ended with error")
79
80
81
82# Execute Shell Commands function
83
84def runCommand(command):
85 command = command.rstrip("\n").strip()
86 osstdout = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
87
88 commandInfo = osstdout.communicate()[0].strip()
89
90 if osstdout.returncode != 0:
91 logging.error("Error while executing Command - '{}'.\nScript Ended with Error : {}".format(command,commandInfo))
92 sys.exit("Error while executing Command - '{}'.\nScript Ended with Error : {}".format(command,commandInfo))
93 else:
94 logging.info("Command - '{}' executed successfully".format(command))
95
96def changeDir(command):
97 command = command.rstrip("\n").strip()
98 try:
99 os.chdir(command)
100 logging.info("Command - cd '{}' executed successfully".format(command))
101 except Exception as e:
102 logging.error("Error while executing Command - 'cd {}'.Please use absolute paths.\nScript Ended with Error : {}".format(command,e))
103 sys.exit("Error while executing Command - 'cd {}'.Please use absolute paths.\nScript Ended with Error : {}".format(command,e))
104
105
106
107# Check release nos. - If new release is not greater than old release then exit
108
109oldReleaseNo=os.popen("basename {}".format(data['oldRelease']).strip()).read().rstrip('\n').strip()
110newReleaseNo=os.popen("basename {}".format(data['newRelease']).strip()).read().rstrip('\n').strip()
111
112logging.info("Old Release No. - " + oldReleaseNo)
113logging.info("New Release No. - " + newReleaseNo)
114
115if int(newReleaseNo.split('.')[0]) > int(oldReleaseNo.split('.')[0]):
116 logging.info("Release nos. are entered are correct")
117elif int(newReleaseNo.split('.')[0]) == int(oldReleaseNo.split('.')[0]):
118 if int(newReleaseNo.split('.')[1]) > int(oldReleaseNo.split('.')[1]):
119 logging.info("Release nos. are entered are correct")
120 elif int(newReleaseNo.split('.')[1]) == int(oldReleaseNo.split('.')[1]):
121 if int(newReleaseNo.split('.')[2]) > int(oldReleaseNo.split('.')[2]):
122 logging.info("Release nos. are entered are correct")
123 else:
124 logging.error("Script ended with error. Wrong Old and New releases entered in {}. Please check.".format(confFile))
125 logging.error("Script ended with error")
126 sys.exit("Script ended with error. Wrong Old and New releases entered in {}. Please check.".format(confFile))
127 else:
128 logging.error("Script ended with error. Wrong Old and New releases entered in {}. Please check.".format(confFile))
129 logging.error("Script ended with error")
130 sys.exit("Script ended with error. Wrong Old and New releases entered in {}. Please check.".format(confFile))
131else:
132 logging.error("Script ended with error. Wrong Old and New releases entered in {}. Please check.".format(confFile))
133 logging.error("Script ended with error")
134 sys.exit("Script ended with error. Wrong Old and New releases entered in {}. Please check.".format(confFile))
135
136# Deployment Starts
137changeDir(data['stagingDir'])
138
139runCommand('git clone -b {} {}'.format(data['gitBranch'],data['gitRepo']))
140runCommand('mkdir -p {}'.format(data['newRelease']))
141if data['cherryPick'].lower() == 'y':
142
143# Check filesToDeployPath
144
145 runCommand("ls {}".format(data['filesToDeployPath'].strip()))
146 runCommand('cp -R {}/* {}'.format(data['oldRelease'],data['newRelease']))
147 changeDir(data['newRelease'])
148
149 with open(data['filesToDeployPath'], 'r') as filesToDeploy:
150 for file in filesToDeploy.readlines():
151 file = file.rstrip("\n").strip()
152 if file:
153 runCommand('cp {}/{}/{} {}'.format(data['stagingDir'],data['gitRepo'].split('/')[-1].split('.')[0],file,file))
154
155 changeDir('{}/hive'.format(data['newRelease']))
156 runCommand("perl -pi -e 's/\\r\\n/\\n/' *.hql")
157 runCommand("sed -i $'s/\\t/ /g' *.hql")
158 changeDir('{}/sh'.format(data['newRelease']))
159 runCommand("perl -pi -e 's/\\r\\n/\\n/' *.sh")
160 runCommand("chmod 755 *.sh")
161 runCommand('ln -sfn {} {}'.format(data['newRelease'],data['currentRelease']))
162
163
164elif data['cherryPick'].lower() == 'n':
165
166 runCommand('cp -R {}/{}/* {}'.format(data['stagingDir'],data['gitRepo'].split('/')[-1].split('.')[0],data['newRelease']))
167 changeDir('{}/hive'.format(data['newRelease']))
168 runCommand("perl -pi -e 's/\\r\\n/\\n/' *.hql")
169 runCommand("sed -i $'s/\\t/ /g' *.hql")
170 changeDir('{}/sh'.format(data['newRelease']))
171 runCommand("perl -pi -e 's/\\r\\n/\\n/' *.sh")
172 runCommand("chmod 755 *.sh")
173 runCommand('ln -sfn {} {}'.format(data['newRelease'],data['currentRelease']))
174
175else:
176 logging.error("Please select whether to cherrypick or not. Please enter the value in confFile at {}".format(confFile))
177 sys.exit("Please select whether to cherrypick or not. Please enter the value in confFile at {}".format(confFile))
178
179# Emptying fileToDeployPath so that it doesn't hamper next deployment
180
181runCommand('> {}'.format(sys.argv[1]))
182runCommand('> {}'.format(data['filesToDeployPath']))
183
184# Purge Log Files Older than 90 days
185
186logging.info("Log Path : " + logpath)
187now = time.time()
188
189for f in os.listdir(logpath):
190 log = os.path.join(logpath,f)
191 if log.endswith('.log'):
192 if os.stat(log).st_mtime < now - 90 * 86400:
193 if os.path.isfile(log):
194 logging.info("Removing log file : {log}".format(log = log))
195 try:
196 os.remove(log)
197 logging.info("Removed log file : {log}".format(log = log))
198 except Exception as e:
199 logging.error("Exception" + e)
200 logging.error("Unable to delete log file : " + log)
201 sys.exit("Script ended with error")
202
203logging.info("Script Ended Successfully")
204
205
206================================================================================================================================
207
208================================================================================================================================
209
210================================================================================================================================
211
212# =============================================================================
213'''
214 File name: executeShellCommands.py
215 Description: Execute Shell Commands
216 Author: Divit Sharma
217 Date created: 05/01/2019
218 Date last modified: 05/01/2019
219 Python Version: 3.7.1 ()
220 Execution : python3 executeShellCommands.py /path/crqAutoDeploy/commandFile.txt
221'''
222# =============================================================================
223# Imports
224# =============================================================================
225
226import sys,os,logging,subprocess,time
227from datetime import date
228from datetime import datetime
229
230# =============================================================================
231# Code
232# =============================================================================
233
234#Logging
235logpath = "/path/crqAutoDeploy/logs"
236timeLog = str(datetime.now().strftime("%Y-%m-%d_%H_%M_%S"))
237print("Script Started at " + timeLog)
238
239logName = '{logpath}/executeShellCommands_{timeLog}.log'.format(timeLog = timeLog, logpath = logpath)
240try:
241 os.popen("ls {}".format(logpath.strip())).read()
242except:
243 sys.exit("Logpath not set")
244
245uid = os.popen("whoami").read().rstrip("\n")
246
247level = logging.INFO
248handlers = [logging.FileHandler(logName), logging.StreamHandler()]
249
250logging.basicConfig(handlers = handlers, format="%(levelname)s - {uid} - %(asctime)s - %(message)s".format(uid=uid),level = level,datefmt='%Y-%m-%d %H:%M:%S')
251
252# Check commandFile exists or not
253if len(sys.argv) == 2:
254
255 commandFile=sys.argv[1]
256
257 logging.info("commandFile : " + commandFile)
258
259else:
260 logging.error("Please check the arguments. Script should have 1 argument i.e. Absolute path of command file")
261 logging.error("Script Ended with Error")
262 sys.exit("Please check the arguments. Script should have 1 argument i.e. Absolute path of command file.\nScript Ended with Error")
263
264
265try:
266 os.popen("ls {}".format(commandFile.strip())).read()
267except:
268 logging.error("Script ended with error. commandFile not found at {}".format(commandFile))
269 logging.error("Script ended with error")
270 sys.exit("Script ended with error. commandFile not found at {}".format(commandFile))
271
272logging.info("commandFile found")
273
274#Run Commands
275
276def runCommand(command):
277 osstdout = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
278
279 commandInfo = osstdout.communicate()[0].strip()
280
281 return (osstdout.returncode,commandInfo)
282
283with open(commandFile, 'r') as commandFile:
284 commandNo = 0
285 for command in commandFile:
286 commandNo = commandNo + 1
287 if command.rstrip("\n"):
288 command = command.rstrip("\n").strip()
289 if command[:3] == 'cd ' :
290 command = command[2:].strip()
291 try:
292 os.chdir(command)
293 logging.info("Command {} - 'cd {}' executed successfully".format(commandNo,command))
294 except Exception as e:
295 logging.error("Error while executing Command {} - 'cd {}'.Please use absolute paths.\nScript Ended with Error : {}".format(commandNo,command,e))
296 sys.exit("Error while executing Command {} - 'cd {}'.Please use absolute paths.\nScript Ended with Error : {}".format(commandNo,command,e))
297 else:
298 status,commandInfo = runCommand(command)
299
300 if status != 0:
301 logging.error("Error while executing Command {} - '{}'.\nScript Ended with Error : {}".format(commandNo,command,commandInfo))
302 sys.exit("Error while executing Command {} - '{}'.\nScript Ended with Error : {}".format(commandNo,command,commandInfo))
303 else:
304 logging.info("Command {} - '{}' executed successfully".format(commandNo,command))
305
306# Emptying commandFile so that it doesn't hamper next deployment
307command = '> {}'.format(sys.argv[1])
308
309status,commandInfo = runCommand(command)
310if status != 0:
311 logging.error("Error while executing Command {} - '{}'.\nScript Ended with Error : {}".format(commandNo,command,commandInfo))
312 sys.exit("Error while executing Command {} - '{}'.\nScript Ended with Error : {}".format(commandNo,command,commandInfo))
313else:
314 logging.info("Command {} - '{}' executed successfully".format(commandNo,command))
315
316# Purge Log Files Older than 90 days
317
318logging.info("Log Path : " + logpath)
319now = time.time()
320
321for f in os.listdir(logpath):
322 log = os.path.join(logpath,f)
323 if log.endswith('.log'):
324 if os.stat(log).st_mtime < now - 90 * 86400:
325 if os.path.isfile(log):
326 logging.info("Removing log file : {log}".format(log = log))
327 try:
328 os.remove(log)
329 logging.info("Removed log file : {log}".format(log = log))
330 except Exception as e:
331 logging.error("Exception" + e)
332 logging.error("Unable to delete log file : " + log)
333 sys.exit("Script ended with error")
334
335
336
337=========================================================================================================================================
338==========================================================================================================================================
339
340
341
342# =============================================================================
343'''
344 File name: radDuplicateFileCheck.py
345 Description: Checks duplicate files in hive table partitions
346 Author: Divit Sharma
347 Date created: 04/04/2019
348 Date last modified: 04/04/2019
349 Python Version: 3.7.1 ()
350 Execution : python3 radDuplicateFileCheck.py
351'''
352# =============================================================================
353# Imports
354# =============================================================================
355
356from elasticsearch import Elasticsearch
357from datetime import date
358from datetime import datetime
359import os
360import time
361from Crypto.Cipher import AES
362import base64
363from ssl import create_default_context
364import logging
365import smtplib
366from email.mime.text import MIMEText
367from email.mime.multipart import MIMEMultipart
368from threading import *
369
370# =============================================================================
371# Code
372# =============================================================================
373
374
375
376#Logging
377try:
378 logpath = "/path/elk/DuplicateFileCheck/logs"
379 os.popen("ls {}".format(logpath.strip())).read()
380except:
381 sys.exit("logpath not set")
382
383timeLog = str(datetime.now().strftime("%Y-%m-%d_%H_%M_%S"))
384
385print("Script Started at " + timeLog)
386
387logName = '{logpath}/radDuplicateFileCheck_{timeLog}.log'.format(timeLog = timeLog, logpath = logpath)
388
389
390uid = os.popen("whoami").read().rstrip("\n")
391
392level = logging.INFO
393handlers = [logging.FileHandler(logName), logging.StreamHandler()]
394
395logging.basicConfig(handlers = handlers, format='%(levelname)s - {uid} - %(message)s'.format(uid=uid), level = level)
396
397#Email
398
399def send_mail(subject, message):
400 username = "hcdss.support@walgreens.com"
401 #from address
402 recipient = "hcdss.support@walgreens.com"
403 #to address
404 #recipient = "divit.sharma@walgreens.com, hcdss.support@walgreens.com"
405
406 msg = MIMEMultipart()
407 msg['From'] = username
408 msg['To'] = recipient
409 msg['Subject'] = subject
410 msg.attach(MIMEText(message))
411
412 try:
413 logging.info('Sending mail to ' + recipient + ' Subject : ' + subject)
414 mailServer = smtplib.SMTP('webmail.walgreens.com', 25)
415 mailServer.ehlo()
416 mailServer.starttls()
417 mailServer.sendmail(username, recipient, msg.as_string())
418 mailServer.close()
419
420 except Exception as e:
421 logging.error("Unable to send mail to " + recipient + ' Subject : ' + subject )
422 logging.error("Exception : " + str(e))
423 logging.error("Script ended with error")
424 sys.exit("Script ended with error")
425
426
427# Check Key File in which encrypted elasticsearch username and password are kept
428
429try:
430 keyFile='/path/elk/radDuplicateFileCheck/keyFile'
431 os.popen("ls {}".format(keyFile.strip())).read()
432except:
433 logging.error("keyFile not found")
434 logging.error("Script ended with error")
435 sys.exit("Script ended with error")
436
437logging.info("keyFile found")
438logging.info("keyFile shoud have entries in this format - key:value")
439
440# Decrypt username and password
441
442data = dict()
443
444with open(keyFile, 'r') as raw_data:
445 for item in raw_data:
446 if item:
447
448 if ':' in item:
449 key,value = item.split(':', 1)
450 data[key]=value.strip()
451 else:
452 logging.error("Unknown Key : " + item)
453 logging.error("Script Ended with Error. Please check key file present at " + keyFile)
454 sys.exit("Script ended with error")
455
456
457secret_key = os.environ["HOSTNAME"]
458
459if (len(secret_key) <=16 ):
460 secret_key = secret_key.rjust(16)
461elif (len(secret_key) <=24 ):
462 secret_key = secret_key.rjust(24)
463elif (len(secret_key) <=32 ):
464 secret_key = secret_key.rjust(32)
465else:
466 logging.error("Script Ended with Error. HOSTNAME : {hostname} length greater than 32. This can't be used as secret key. Please change the secret_key in code.").format(hostname = secretkey)
467 sys.exit("Script ended with error")
468
469
470
471cipher = AES.new(secret_key,AES.MODE_ECB)
472
473try:
474 uname = cipher.decrypt(base64.b64decode(data['uname'])).strip().decode('utf-8')
475except:
476 logging.error("Unable to decrypt elasticsearch username")
477 logging.error("Script ended with error")
478 sys.exit("Script ended with error")
479
480logging.info("Successfully decrypted elasticsearch username")
481
482try:
483 password = cipher.decrypt(base64.b64decode(data['password'])).strip().decode('utf-8')
484except:
485 logging.error("Unable to decrypt elasticsearch password")
486 logging.error("Script ended with error")
487 sys.exit("Script ended with error")
488
489logging.info("Successfully decrypted elasticsearch password")
490
491
492# Make connection with elasticsearch
493
494context = create_default_context(cafile="/path/elk/WAG_CA_root.crt")
495
496try:
497 es = Elasticsearch(['hostname'], port=9200, http_auth=(uname, password),scheme="https", ssl_context=context)
498except:
499 logging.error("Unable to make connection with elasticsearch server")
500 logging.error("Script ended with error")
501 sys.exit("Script ended with error")
502
503logging.info("Successfully connected with elasticsearch server :-")
504
505logging.info(es.info())
506
507
508# Tables for which duplicate files have to be checked
509try:
510 tables='/path/elk/DuplicateFileCheck/tables.txt'
511 os.popen("ls {}".format(tables.strip())).read()
512except:
513 logging.error("keyFile not found")
514 logging.error("Script ended with error")
515 sys.exit("Script ended with error")
516
517logging.info("tables.txt found : " + tables)
518
519# Read duplicate files from hadoop directory for each table
520
521def duplicateFileCheck(table):
522 try:
523 fileAll = os.popen("hadoop fs -ls /path/{table}/data/*/* 2>/dev/null".format(table = table) + " | awk '{print $NF}' | xargs -n 1 basename 2>/dev/null | sort | uniq -d").read().rstrip("\n").split('\n')
524 except Exception as e:
525 logging.error(e)
526 logging.error("Unable to fetch duplicate files for table" + table)
527 sys.exit("Script ended with error")
528
529
530 if fileAll == ['']:
531 logging.info("No duplicate files found for {table}".format(table = table))
532 return None
533
534 else:
535 logging.info("Duplicate files for {table} are : ".format(table = table) + ','.join(fileAll))
536
537 today = str(date.today())
538 logging.info("Date : " + today)
539 estimestamp = str(datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
540 logging.info("Elasticsearch Index Time : " + estimestamp)
541
542 directoryAll = []
543
544 for file in fileAll:
545 if file != "":
546 try:
547 directory = os.popen("hadoop fs -ls /path/{table}/data/*/{file}".format(file = file, table = table) + " | awk '{print $NF}'").read().rstrip("\n").split('\n')
548 except Exception as e:
549 logging.error("Unable to fetch directory for duplicate file" + file)
550 sys.exit("Script ended with error")
551
552 directoryAll = directoryAll + directory
553
554 body = {"@timestamp": estimestamp, "Date": today, "Duplicate File": file, "Directory": directory}
555 logging.info("Document body : " + str(body))
556
557
558 #Index data into Elasticsearch
559
560 try:
561 resp = es.index(index="rad_duplicate_file_check", doc_type="_doc", body=body, id=table+file)
562
563 except Exception as e:
564 logging.error("Unable to index below document into elasticsearch :- \n" + ','.join(resp))
565 sys.exit("Script ended with error")
566
567 logging.info("Document Indexed Successfully :- ")
568 logging.info(resp)
569
570
571
572
573 # Alert on receiving duplicate file
574
575 if directoryAll:
576
577 subject = "{date} | RAD Duplicate File Check | Duplicate file found for table {table}".format( table = table, date = today)
578
579 message = 'Duplicate files {fileAll} found for table {table} at below directories : \n\n{directoryAll}. \n\n\nPlease take necessary action.'.format( fileAll = fileAll, table = table, directoryAll = ', '.join(directoryAll))
580
581 send_mail(subject, message)
582
583
584# Multithreading - Each table check is a new thread
585
586threads = []
587
588with open(tables, 'r') as tables:
589 for table in tables.readlines():
590 table = table.rstrip("\n")
591 t= Thread(target=duplicateFileCheck, args=(table,))
592 threads.append(t)
593 t.start()
594
595for thread in threads:
596 thread.join()
597
598
599# Delete older(more than 7 days) Elasticsearch Documents
600
601time.sleep(10)
602
603try:
604 es_delete_docs= es.delete_by_query(index='duplicate_file_check',body={"query": {"range" : {"@timestamp" : {"lt" : "now-7d","format": "yyyy-MM-dd HH:mm:ss"}}}})
605
606except Exception as e:
607 logging.error("Unable to delete below document from elasticsearch :- \n" + ','.join(es_delete_docs))
608 sys.exit("Script ended with error")
609
610logging.info("Documents deleted successfully :- ")
611logging.info(es_delete_docs)
612
613
614
615# Purge Log Files Older than 30 days
616
617logging.info("Log Path : " + logpath)
618now = time.time()
619
620for f in os.listdir(logpath):
621 log = os.path.join(logpath,f)
622 if log.endswith('.log'):
623 if os.stat(log).st_mtime < now - 30 * 86400:
624 if os.path.isfile(log):
625 logging.info("Removing log file : {log}".format(log = log))
626 try:
627 os.remove(log)
628 logging.info("Removed log file : {log}".format(log = log))
629 except Exception as e:
630 logging.error("Exception" + e)
631 logging.error("Unable to delete log file : " + log)
632 sys.exit("Script ended with error")
633
634logging.info("Script Ended Successfully")
635
636
637logging.info("Script Ended successfully")
638
639
640
641
642=======================================================================================================================================================
643========================================================================================================================================================
644
645# =============================================================================
646'''
647 File name: encryptStrings.py
648 Description: Generates encrypted values for a string provided
649 Author: Divit Sharma
650 Date created: 04/04/2019
651 Date last modified: 04/04/2019
652 Python Version: 3.7.1 ()
653 Execution : python3 encryptStrings.py
654'''
655# =============================================================================
656# Imports
657# =============================================================================
658from Crypto.Cipher import AES
659import base64
660import getpass
661import os
662
663# =============================================================================
664# Code
665# =============================================================================
666
667str = getpass.getpass('Enter the string to encrypt : ').strip()
668
669if (len(str)%16 == 0):
670 str = str.rjust(((int(len(str)/16))*16))
671else:
672 str = str.rjust((((int(len(str)/16)+1))*16))
673
674
675secret_key= os.environ["HOSTNAME"]
676
677if (len(secret_key) <=16 ):
678 secret_key = secret_key.rjust(16)
679elif (len(secret_key) <=24 ):
680 secret_key = secret_key.rjust(24)
681elif (len(secret_key) <=32 ):
682 secret_key = secret_key.rjust(32)
683else:
684 print("HOSTNAME length greater than 32. This can't be used as secret key. Please change the secret_key in code.")
685
686
687cipher = AES.new(secret_key,AES.MODE_ECB)
688print (base64.b64encode(cipher.encrypt(str)).decode('utf-8'))
689
690
691========================================================================================================================================
692==========================================================================================================================================
693
694
695
696#!/usr/local/bin/python
697
698#################################################################################################################
699# SCRIPT NAME: excelToCsvConverter.py
700# AUTHOR NAME: Divit Sharma
701# CREATION DATE: FEB 2019
702# CURRENT REVISION NO: 1
703#
704# DESCRIPTION: This is a generic script which converts xlsx to csv
705# DEPENDENCIES: NA
706# Input: 1.Input xlsx file 2.Output csv file 3. Sheet Name 4. Datatype file
707#-------------------------------------------------------------------------------------+
708# M A I N T E N A N C E H I S T O R Y
709#-------------------------------------------------------------------------------------+
710# Revision| Description | Name | Date
711#---------+----------------------------------------+----------------------+-----------+
712# 1.0 | Initial release. | Divit Sharma | 2019-02-07
713#---------+----------------------------------------+----------------------+-----------+
714
715
716
717import sys
718import pandas as pd
719
720print("Inside Python Script. Excel to CSV script started")
721
722if len(sys.argv) == 5:
723 input_xlsx_file=sys.argv[1]
724 output_csv_file=sys.argv[2]
725 sheet_name=sys.argv[3]
726 datatype=sys.argv[4]
727
728 print("Input xlsx file : " + input_xlsx_file)
729 print("Output csv file : " + output_csv_file)
730 print("Sheetname : " + sheet_name)
731 print("Datatype File : " + datatype)
732
733else:
734 print("Please check the arguments. Script should have 4 arguments i.e. Absolute path of Input xlsx file,Output csv file, sheet name and datatype file")
735 print("Script Ended with Error")
736 sys.exit()
737
738data = dict()
739
740with open(datatype, 'r') as raw_data:
741 for item in raw_data:
742 if ':' in item:
743 key,value = item.split(':', 1)
744 data[key]=value.strip()
745 else:
746 print("Unknown Column name or datatype : " + item)
747 print("Script Ended with Error. Please check datatype file present at " + datatype)
748 sys.exit()
749
750data_xlsx = pd.read_excel(input_xlsx_file,sheet_name,dtype = data)
751
752for key,value in data.items():
753 if value == 'str':
754 data_xlsx[key] = data_xlsx[key].replace('nan','')
755
756print("Excel to CSV conversion started. Please wait.")
757
758data_xlsx.to_csv(output_csv_file, sep='|',quotechar='"',header=False,index=False,encoding='utf-8')
759
760print("Excel to CSV conversion completed. Python Script Completed")
761
762
763
764==========================================================================================================================================
765==============================================================================================================================================
766
767 remove_field => [ "host", "message", "path", "@timestamp", "@version" ]
768
769 POST index_name/_delete_by_query
770{
771"query": {
772"range" : {
773"load_dt" : {
774"lt" : "now-16d",
775"format": "MMM DD, YYYY"
776}
777}
778}
779}
780
781
782
783
784
785
786nohup ./bin/logstash -f /logstash_homepath/config/conf.d/ -r 2>&1 &
787
788nohup ./filebeat -c /filebeat_homepath/filebeat_logstash/filebeat_logstash.yml -path.home /filebeat_homepath/filebeat_logstash -path.config /filebeat_homepath/filebeat_logstash -path.data /filebeat_homepath/filebeat_logstash -path.logs /filebeat_homepath/filebeat_logstash -e 2>&1 &
789
790
791
792nohup ./filebeat -c /filebeat_homepath/filebeat_elasticsearch/filebeat_elasticsearch.yml -path.home /filebeat_homepath/filebeat_elasticsearch/ -path.config /filebeat_homepath/filebeat_elasticsearch/ -path.data /filebeat_homepath/filebeat_elasticsearch/ -path.logs /filebeat_homepath/filebeat_elasticsearch/ -e 2>&1 &
793
794
795Keystore logstash
796/logstash-home/bin/logstash-keystore create
797Use value in config using :
798
799 hosts => "${es_host}"
800 user => "${es_user}"
801 password => "${es_password}"
802
803
804 =====================
805
806 curl --insecure --user username:password -XGET "https://elastichostaddress:9200"
807
808 Metricbeat :
809
810Load/Setup default dashboards:
811cd /metricbeat_home/metricbeat-6.3.2-linux-x86_64
812
813./metricbeat setup
814
815
816Run metricbeat without loading default dashboard:
817nohup ./metricbeat run -e --setup 2>&1 &
818
819
820 ============================================
821
822 GIT
823
824
825git clone -b /feature/featurename http://repositoryname.git
826
827git config --global user.name 'Divit Sharma'
828git config --global user.email 'divit.sharma@gmail.com'
829git status
830git add --all
831git commit -m "Add commit message here"
832git push origin feature/featurename
833
834
835Solve merge issues
836
837git pull origin branch-with-more-content(like release/3.7.0)
838git push origin branch-withlesscontent(like feature)
839
840
841================================================================================================================================
842
843
844-----------------------witohut ssl----------------
845
846es = Elasticsearch(['hostname'], port=9250, http_auth=('username', 'password'),scheme="https", ca_certs=False, verify_certs=False)
847
848
849result = es.search(index="shell_test", body={"query": {"match_all": {}}})
850print(result)
851----------------with ssl----------------------------------------------
852
853from elasticsearch import Elasticsearch
854from ssl import create_default_context
855
856context = create_default_context(cafile="/abc.crt")
857es = Elasticsearch(['hostname'], port=9250, http_auth=('username', 'password'),scheme="https", ssl_context=context)
858
859result = es.search(index="shell_test", body={"query": {"match_all": {}}})
860
861
862===============================================================================================================================================
863
864
865
866input {
867 jdbc {
868 type => "job_status"
869 jdbc_connection_string => "${jdbc_string}" #jdbc:oracle:thin:@oraclehost:port/servicename
870 jdbc_user => "${oracle_dae_user}"
871 jdbc_password => "${oracle_dae_pwd}"
872 schedule => "0,10,20,30,40,50 * * * *"
873 use_column_value => true
874 tracking_column => "end_dt_tm"
875 tracking_column_type => timestamp
876 jdbc_validate_connection => true
877 jdbc_driver_library => "/usr/share/java/ojdbc8.jar"
878 jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
879 statement => "SELECT JOB_ID,START_DT_TM, END_DT_TM FROM (SELECT * FROM TABLE_JOB WHERE PROJ_ID= 'abc' AND END_DT_TM > :sql_last_value ORDER BY END_DT_TM) UNION SELECT JOB_ID,START_DT_TM, END_DT_TM FROM TABLE_JOB WHERE PROJ_ID= 'abc' AND END_DT_TM IS NULL"
880 last_run_metadata_path => "/logstash_home/config/logstash_last_run/.job_status_last_run"
881 }
882}
883
884filter{
885
886}
887
888output {
889 if [type] == "job_status" {
890 elasticsearch {
891 hosts => "https://hostnameaddress:9200"
892 user => "username"
893 password => "password"
894 index => "job_status"
895 document_id => "%{batch_id}-%{job_id}"
896
897 }
898 }
899}