· 4 years ago · Jan 12, 2021, 07:14 PM
1#!/usr/bin/env python3
2#-*- coding: utf-8 -*-
3# Import module
4import io
5import os
6import time
7import pandas
8import argparse
9import csv
10from pandas import ExcelWriter
11from pymongo import MongoClient, errors
12from bson.objectid import ObjectId
13from datetime import datetime
14from pathlib import Path
15from colorama import init, Fore
16init()
17
18def welcomebanner():
19 # Print the welcome banner
20 print(Fore.CYAN)
21 message = "* EC2 MongoDB *"
22 banner(message, "*")
23 print(Fore.RESET)
24
25def endbanner():
26 print(Fore.CYAN)
27 message = "* EC2 MongoDB Operations Are Complete *"
28 banner(message, "*")
29 print(Fore.RESET)
30
31def banner(message, border="-"):
32 line = border * len(message)
33 print(line)
34 print(message)
35 print(line)
36
37def exit_program():
38 endbanner()
39 exit()
40
41def is_digit(check_input):
42 if check_input.isdigit():
43 return True
44 return False
45
46def initialize(interactive, aws_account):
47 # Set the date
48 today = datetime.today()
49 today = today.strftime("%m-%d-%Y")
50 # Set the input file
51 aws_env_list = os.path.join('..', '..', 'source_files', 'aws_accounts_list', 'aws_accounts_list.csv')
52 return today, aws_env_list
53
54def read_account_info(aws_env_list):
55 account_names = []
56 account_numbers = []
57 with open(aws_env_list) as csv_file:
58 csv_reader = csv.reader(csv_file, delimiter=',')
59 next(csv_reader)
60 for row in csv_reader:
61 account_name = str(row[0])
62 account_number = str(row[1])
63 account_names.append(account_name)
64 account_numbers.append(account_number)
65 return account_names, account_numbers
66
67def select_account(options, aws_env_list):
68 ## Select the account
69 if options.account_name:
70 aws_account = options.account_name
71 else:
72 print(Fore.YELLOW)
73 aws_account = input("Enter the name of the AWS account you'll be working in: ")
74 print(Fore.RESET)
75 aws_account_number = find_account_number(aws_account, aws_env_list)
76 return aws_account, aws_account_number
77
78def find_account_number(aws_account,aws_env_list):
79 account_names, account_numbers = read_account_info(aws_env_list)
80 for (my_aws_account, my_aws_account_number) in zip(account_names, account_numbers):
81 if my_aws_account == aws_account:
82 aws_account_number = my_aws_account_number
83 if aws_account == "all":
84 aws_account_number = '1234567891011'
85 return aws_account_number
86
87def arguments():
88 parser = argparse.ArgumentParser(description="This is a program that provides a text interface to MongoDB.")
89
90 parser.add_argument(
91 "-n",
92 "--account_name",
93 type = str,
94 default = None,
95 nargs = "?",
96 help = "Name of the AWS account you'll be working in")
97
98 options = parser.parse_args()
99 return options
100
101def create_directories():
102
103 ## Set source and output file directories
104 source_files_path = Path('..', '..', 'source_files', 'aws_accounts_list')
105 output_files_path = Path('..', '..', 'output_files', 'aws_instance_list')
106
107 os.makedirs(source_files_path, exist_ok=True)
108 os.makedirs(output_files_path, exist_ok=True)
109
110 # Create output subdirectories
111 folders = ['csv','excel','html', 'json']
112 for folder in folders:
113 full_path = os.path.join(output_files_path,folder)
114 os.makedirs(full_path, exist_ok=True)
115
116def set_test_dict():
117 mydict = { "AWS Account": "company-lab", "Account Number": "12345678910", "Name": "bastion001",
118"Instance ID": "i-07aaef3b7167d592a", "AMI ID": "ami-07fd81f1ecf6cf387", "Volumes": "vol-09d6d898db4af132a",
119"Private IP": "10.238.3.165", "Public IP": "xx.xx.xx.xx", "Private DNS": "ip-10-238-3-165.ec2.internal",
120"Availability Zone": "us-east-1a", "VPC ID": "vpc-00de11103235ec567", "Type": "t3.small", "Key Pair Name": "ccmi-vzn-int01", "Instance State": "running", "Launch Date": "September 10 2019"}
121 return mydict
122
123def connect_db():
124 try:
125 myclient = MongoClient(
126 host = "mongodb://localhost:27017/",
127 serverSelectionTimeoutMS = 3000 # 3 second timeout
128 )
129 except errors.ServerSelectionTimeoutError as e:
130 # set the client instance to 'None' if exception
131 myclient = None
132 # catch pymongo.errors.ServerSelectionTimeoutError
133 print ("pymongo ERROR:", e)
134 return myclient
135
136def set_db(instance_col=None):
137 if __name__ == "__main__":
138 message = "* Select a MongoDB Database *"
139 print(Fore.CYAN)
140 banner(message, "*")
141 print(Fore.RESET)
142 myclient = connect_db()
143 today = datetime.today()
144 today = today.strftime("%m%d%Y")
145 if __name__ == "__main__":
146 print(Fore.CYAN + "Available MongoDB Databases:")
147 if myclient != None:
148 # the list_database_names() method returns a list of strings
149 database_names = myclient.list_database_names()
150 counter = 1
151 for db in database_names:
152 message = str(counter) + ". " + db
153 print(message)
154 counter = counter + 1
155 print ("There are", len(database_names), "databases.\n")
156 print(f"Please select a database. Enter a number 1 through {len(database_names)}.")
157 choice = input("Enter a number: ")
158 if is_digit(choice) == True:
159 if int(choice) > counter:
160 print("Wrong selection.")
161 set_db()
162 choice = int(choice)
163 choice = choice - 1
164 mydb = myclient[database_names[choice]]
165 mydb_name = database_names[choice]
166 instance_col = "ec2_list_" + today
167 instance_col = mydb[instance_col]
168 print(f"You've selected: {database_names[choice]}\n")
169 else:
170 print("Must enter a digit. Try again.\n")
171 else:
172 mydb = myclient["aws_inventories"]
173 mydb_name = "aws_inventories"
174 instance_col = "ec2_list_" + today
175 instance_col = mydb[instance_col]
176 return mydb, mydb_name, instance_col
177
178def create_mongodb(mydict):
179 myclient = connect_db()
180 message = f"* Create new MongoDB *"
181 banner(message, border="*")
182 print("\n")
183 newdb = input("Enter the name of a new mongo database: ")
184 dblist = myclient.list_database_names()
185 if newdb in dblist:
186 print("The database exists.")
187 main()
188 else:
189 try:
190 mydb = myclient[newdb]
191 mycol = mydb["testColumn"]
192 mycol.insert_one(mydict)
193 message = f"Succeeded in creating: {newdb}"
194 banner(message)
195 except Exception as e:
196 print(f"MongoDB Database creation failed with: {e}")
197
198def drop_mongodb():
199 message = "* Drop MongoDB *"
200 banner(message, "*")
201 myclient = connect_db()
202 today = datetime.today()
203 today = today.strftime("%m%d%Y")
204 if myclient != None:
205 # the list_database_names() method returns a list of strings
206 database_names = myclient.list_database_names()
207 counter = 1
208 for db in database_names:
209 message = str(counter) + ". " + db
210 print(message)
211 counter = counter + 1
212 print ("There are", len(database_names), "databases.\n")
213 db_names_before_drop = myclient.list_database_names()
214 print ("db count BEFORE drop:", len(db_names_before_drop))
215 print(f"Please select a database. Enter a number 1 through {len(database_names)}.")
216 choice = input("Enter a number: ")
217 if is_digit(choice) == True:
218 if int(choice) > counter:
219 print("Wrong selection.")
220 set_db()
221 choice = int(choice)
222 choice = choice - 1
223 dropdb = myclient[database_names[choice]]
224 instance_col = "ec2_list_" + today
225 instance_col = dropdb[instance_col]
226 print(f"You've selected: {database_names[choice]}\n")
227 else:
228 print("Must enter a digit. Try again.\n")
229 # check if a collection exists
230 col_exists = instance_col in dropdb.list_collection_names()
231 print ("Some Collection exists:", col_exists) # will print True or False
232 # call MongoDB client object"s drop_database() method to delete a db
233 myclient.drop_database(dropdb) # pass db name as string
234 time.sleep(5)
235 # get all of the database names
236 db_names_after_drop = myclient.list_database_names()
237 print ("db count AFTER drop:", len(db_names_before_drop))
238 diff = len(db_names_before_drop) - len(db_names_after_drop)
239 print ("difference:", diff)
240
241def insert_doc(mydict):
242 _, _, instance_col = set_db()
243 mydict["_id"] = ObjectId()
244 instance_doc = instance_col.insert_one(mydict)
245 if __name__ == "__main__":
246 message = "* MongoDB Insert Document *"
247 banner(message, "*")
248 message = f"MongoDB record inserted: {instance_doc.inserted_id}"
249 banner(message)
250 return instance_doc
251
252def mongo_select_all():
253 _, mydb_name, instance_col = set_db()
254 instance_list = list(instance_col.find())
255 if __name__ == "__main__":
256 message = f"* Print DB Documents in {mydb_name} *"
257 banner(message, border="*")
258 print("\n")
259 if not instance_list:
260 message = f"The database: {mydb_name} has no entries."
261 banner(message)
262 else:
263 message = f"The databse: {mydb_name} has {len(instance_list)} entries."
264 for data in instance_list:
265 print(data)
266 print("\n")
267 return instance_list
268
269def mongo_export_to_file(interactive, aws_account, aws_account_number,instance_col=None):
270 create_directories()
271 today = datetime.today()
272 today = today.strftime("%m-%d-%Y")
273 if not instance_col:
274 _, _, instance_col = set_db()
275 time.sleep(10)
276 # make an API call to the MongoDB server
277 if interactive == 0:
278 mongo_docs = instance_col.find({})
279 else:
280 mongo_docs = instance_col.find({"Account Number": aws_account_number})
281 # Convert the mongo docs to a DataFrame
282 docs = pandas.DataFrame(mongo_docs)
283 # Discard the Mongo ID for the documents
284 docs.pop("_id")
285 if __name__ == "__main__":
286 print("Choose a file format")
287 print("1. CSV")
288 print("2. JSON")
289 print("3. HTML")
290 print("4. Excel")
291 choice = input("Enter a number 1-4: ")
292 choice = int(choice)
293 else:
294 choice = 1
295 if choice == 1:
296 if __name__ == "__main__":
297 # export MongoDB documents to CSV
298 csv_export = docs.to_csv(sep=",") # CSV delimited by commas
299 print ("\nCSV data:", csv_export)
300 # Set the CSV output directory
301 output_dir = os.path.join("..", "..", "output_files", "aws_instance_list", "csv", "")
302 if interactive == 1:
303 output_file = os.path.join(output_dir, "aws-instance-list-" + aws_account + "-" + today +".csv")
304 else:
305 output_file = os.path.join(output_dir, "aws-instance-master-list-" + today +".csv")
306
307 # export MongoDB documents to a CSV file, leaving out the row "labels" (row numbers)
308 docs.to_csv(output_file, ",", index=False) # CSV delimited by commas
309 elif choice == 2:
310 if __name__ == "__main__":
311 json_export = docs.to_json() # return JSON data
312 print ("\nJSON data:", json_export)
313 # Set the JSON output directory
314 output_dir = os.path.join("..", "..", "output_files", "aws_instance_list", "json", "")
315 if interactive == 1:
316 output_file = os.path.join(output_dir, "aws-instance-list-" + aws_account + "-" + today +".json")
317 else:
318 output_file = os.path.join(output_dir, "aws-instance-master-list" + today +".json")
319 # export MongoDB documents to a CSV file, leaving out the row "labels" (row numbers)
320 docs.to_json(output_file)
321 elif choice == 3:
322 html_str = io.StringIO()
323 # export as HTML
324 docs.to_html(
325 buf=html_str,
326 classes="table table-striped"
327 )
328 if __name__ == "__main__":
329 # print out the HTML table
330 print (html_str.getvalue())
331 # Set the HTML output directory
332 output_dir = os.path.join("..", "..", "output_files", "aws_instance_list", "html", "")
333 if interactive == 1:
334 output_file = os.path.join(output_dir, "aws-instance-list-" + aws_account + "-" + today +".html")
335 else:
336 output_file = os.path.join(output_dir, "aws-instance-master-list" + today + ".html")
337 # save the MongoDB documents as an HTML table
338 docs.to_html(output_file)
339 elif choice == 4:
340 # Set the Excel output directory
341 output_dir = os.path.join("..", "..", "output_files", "aws_instance_list", "excel", "")
342 time.sleep(5)
343 if interactive == 1:
344 output_file = os.path.join(output_dir, "aws-instance-list-" + aws_account + "-" + today + ".xlsx")
345 else:
346 output_file = os.path.join(output_dir, "aws-instance-master-list" + today + ".xlsx")
347 # export MongoDB documents to a Excel file, leaving out the row "labels" (row numbers)
348 writer = ExcelWriter(output_file)
349 docs.to_excel(writer,"EC2 List",index=False)
350 writer.save()
351 if __name__ == "__main__":
352 exit = input("Exit program (y/n): ")
353 if exit.lower() == "y" or exit.lower() == "yes":
354 exit_program()
355 else:
356 main()
357
358def clear_db():
359 _, _, instance_col = set_db()
360 message = f"* Clear the DB *"
361 banner(message, border="*")
362 print(f"This command empties the database.\n")
363 try:
364 x = instance_col.delete_many({})
365 except Exception as e:
366 print(f"An error has occurred: {e}")
367 print(x.deleted_count, "documents deleted.")
368
369def delete_from_collection(aws_account_number):
370 _, _, instance_col = set_db()
371 if __name__ == "__main__":
372 message = f"* Clear old entries *"
373 banner(message, border="*")
374 print(f"This command clears old entries the database.\n")
375 aws_account_number = input("Enter an AWS account number: ")
376 try:
377 #instance_col.remove({"Account Number": aws_account_number});
378 instance_col.delete_many({"Account Number": aws_account_number})
379 except Exception as e:
380 print(f"An error has occurred: {e}")
381
382def print_db_names():
383 myclient = connect_db()
384 message = f"* Print DB Names *"
385 banner(message, border="*")
386 print("The database names are:")
387 if myclient != None:
388 # the list_database_names() method returns a list of strings
389 database_names = myclient.list_database_names()
390 counter = 1
391 for db in database_names:
392 message = str(counter) + ". " + db
393 print(message)
394 counter = counter + 1
395 print ("There are", len(database_names), "databases.")
396
397def print_collections():
398 myclient = connect_db()
399 message = f"* Print DB Collections *"
400 banner(message, border="*")
401 print(f"This command prints the database collection names.\n")
402 if myclient != None:
403 # the list_database_names() method returns a list of strings
404 database_names = myclient.list_database_names()
405 print ("There are", len(database_names), "databases.")
406 for db_num, db in enumerate(database_names):
407 print ("\nGetting collections for database:", db, "--", db_num)
408 collection_names = myclient[db].list_collection_names()
409 print ("The MongoDB database returned", len(collection_names), "collections.")
410 # iterate over the list of collection names
411 for col_num, col in enumerate(collection_names):
412 print (col, "--", col_num)
413
414def print_reports(interactive,aws_account,aws_account_number):
415 set_db(instance_col=None)
416 inputDate = input("Enter the date in format 'dd/mm/yyyy': ")
417 day,month,year = inputDate.split('/')
418 isValidDate = True
419 try:
420 datetime(int(year),int(month),int(day))
421 except ValueError :
422 isValidDate = False
423 print_reports(interactive,aws_account,aws_account_number)
424
425 if(isValidDate) :
426 print(f"Input date is valid: {inputDate}")
427 format= "%m%d%Y"
428 inputDate = datetime.strptime(inputDate,"%m/%d/%Y")
429 inputDate = inputDate.strftime(format)
430 else:
431 print(f"Input date is not valid: {inputDate}")
432 print_reports(interactive,aws_account,aws_account_number)
433 myclient = connect_db()
434 mydb = myclient["aws_inventories"]
435 instance_col = "ec2_list_" + inputDate
436 instance_col = mydb[instance_col]
437 mongo_export_to_file(interactive, aws_account, aws_account_number,instance_col)
438
439
440 #print(f"New date format: {inputDate}")
441 #print_reports()
442
443def menu():
444 message = "Main Menu"
445 banner(message)
446 print(Fore.CYAN + "Your available actions: ")
447 print("1. Create new MongoDB Database")
448 print("2. Drop MongoDB Database")
449 print("3. Do a test insert to the DB")
450 print("4. Clear the DB")
451 print("5. Remove accounts from the DB.")
452 print("6. Print the DB")
453 print("7. Print DB Names")
454 print("8. Print collections")
455 print("9. Export MongoDB to file")
456 print("10. Print Reports")
457 print("11. Exit ec2 mongo")
458 print("\n")
459
460def main():
461 options = arguments()
462 welcomebanner()
463 mydict = set_test_dict()
464 if __name__ == "__main__":
465 print(Fore.YELLOW)
466 aws_accounts_answer = input("Work in one or all accounts: ")
467 print(Fore.RESET)
468 # Set interacive variable to indicate one or many accounts
469 if aws_accounts_answer.lower() == "one" or aws_accounts_answer.lower() == "1":
470 interactive = 1
471 else:
472 interactive = 0
473 aws_account = ''
474 _, aws_env_list = initialize(interactive, aws_account)
475 menu()
476 option = input("Enter the option: ")
477 option = int(option)
478 # 1. Create a MongoDB database
479 if option == 1:
480 create_mongodb(mydict)
481 main()
482 # 2. Drop a MongoDB Database
483 elif option == 2:
484 drop_mongodb()
485 main()
486 # 3. Do a test insert to the DB
487 elif option == 3:
488 insert_doc(mydict)
489 main()
490 # 4. Clear the DB"
491 elif option == 4:
492 clear_db()
493 main()
494 # 5. Remove accounts from the DB.
495 elif option == 5:
496 _, aws_account_number = select_account(options, aws_env_list)
497 delete_from_collection(aws_account_number)
498 main()
499 # 6. Print the DB
500 elif option == 6:
501 mongo_select_all()
502 main()
503 # 7. Print DB Names
504 elif option == 7:
505 print_db_names()
506 main()
507 # 8. Print collections
508 elif option == 8:
509 print_collections()
510 main()
511 # 9. Export MongoDB to file
512 elif option == 9:
513 if aws_accounts_answer == "all":
514 aws_account = "all"
515 aws_account_number = "123456789101"
516 else:
517 aws_account, aws_account_number = select_account(options, aws_env_list)
518 mongo_export_to_file(interactive, aws_account, aws_account_number)
519 main()
520 # 10 Print Reports
521 elif option == 10:
522 if aws_accounts_answer == "all":
523 aws_account = "all"
524 aws_account_number = "123456789101"
525 else:
526 aws_account, aws_account_number = select_account(options, aws_env_list)
527 print_reports(interactive,aws_account,aws_account_number)
528 # 11. Exit ec2 mongo
529 elif option == 11:
530 exit_program()
531 # Invalid Input
532 else:
533 message = "That is not a valid option."
534 banner(message)
535 main()
536
537if __name__ == "__main__":
538 main()