· 4 years ago · Jan 12, 2021, 05:26 PM
1#!/usr/bin/env python3
2#-*- coding: utf-8 -*-
3# Import module
4import io
5import os
6import time
7import pandas
8import argparse
9import csv
10from pandas import ExcelWriter
11from pymongo import MongoClient, errors
12from bson.objectid import ObjectId
13from datetime import datetime
14from pathlib import Path
15from colorama import init, Fore
16init()
17
18def welcomebanner():
19 # Print the welcome banner
20 print(Fore.CYAN)
21 message = "* EC2 MongoDB *"
22 banner(message, "*")
23 print(Fore.RESET)
24
25def endbanner():
26 print(Fore.CYAN)
27 message = "* EC2 MongoDB Operations Are Complete *"
28 banner(message, "*")
29 print(Fore.RESET)
30
31def banner(message, border="-"):
32 line = border * len(message)
33 print(line)
34 print(message)
35 print(line)
36
37def exit_program():
38 endbanner()
39 exit()
40
41def is_digit(check_input):
42 if check_input.isdigit():
43 return True
44 return False
45
46def initialize(interactive, aws_account):
47 # Set the date
48 today = datetime.today()
49 today = today.strftime("%m-%d-%Y")
50 # Set the input file
51 aws_env_list = os.path.join('..', '..', 'source_files', 'aws_accounts_list', 'aws_accounts_list.csv')
52 return today, aws_env_list
53
54def read_account_info(aws_env_list):
55 account_names = []
56 account_numbers = []
57 with open(aws_env_list) as csv_file:
58 csv_reader = csv.reader(csv_file, delimiter=',')
59 next(csv_reader)
60 for row in csv_reader:
61 account_name = str(row[0])
62 account_number = str(row[1])
63 account_names.append(account_name)
64 account_numbers.append(account_number)
65 return account_names, account_numbers
66
67def select_account(options, aws_env_list):
68 ## Select the account
69 if options.account_name:
70 aws_account = options.account_name
71 else:
72 print(Fore.YELLOW)
73 aws_account = input("Enter the name of the AWS account you'll be working in: ")
74 print(Fore.RESET)
75 aws_account_number = find_account_number(aws_account, aws_env_list)
76 return aws_account, aws_account_number
77
78def find_account_number(aws_account,aws_env_list):
79 account_names, account_numbers = read_account_info(aws_env_list)
80 for (my_aws_account, my_aws_account_number) in zip(account_names, account_numbers):
81 if my_aws_account == aws_account:
82 aws_account_number = my_aws_account_number
83 if aws_account == "all":
84 aws_account_number = '1234567891011'
85 return aws_account_number
86
87def arguments():
88 parser = argparse.ArgumentParser(description="This is a program that provides a text interface to MongoDB.")
89
90 parser.add_argument(
91 "-n",
92 "--account_name",
93 type = str,
94 default = None,
95 nargs = "?",
96 help = "Name of the AWS account you'll be working in")
97
98 options = parser.parse_args()
99 return options
100
101def create_directories():
102
103 ## Set source and output file directories
104 source_files_path = Path('..', '..', 'source_files', 'aws_accounts_list')
105 output_files_path = Path('..', '..', 'output_files', 'aws_instance_list')
106
107 os.makedirs(source_files_path, exist_ok=True)
108 os.makedirs(output_files_path, exist_ok=True)
109
110 # Create output subdirectories
111 folders = ['csv','excel','html', 'json']
112 for folder in folders:
113 full_path = os.path.join(output_files_path,folder)
114 os.makedirs(full_path, exist_ok=True)
115
116def set_test_dict():
117 mydict = { "AWS Account": "company-lab", "Account Number": "12345678910", "Name": "bastion001",
118"Instance ID": "i-07aaef3b7167d592a", "AMI ID": "ami-07fd81f1ecf6cf387", "Volumes": "vol-09d6d898db4af132a",
119"Private IP": "10.238.3.165", "Public IP": "xx.xx.xx.xx", "Private DNS": "ip-10-238-3-165.ec2.internal",
120"Availability Zone": "us-east-1a", "VPC ID": "vpc-00de11103235ec567", "Type": "t3.small", "Key Pair Name": "ccmi-vzn-int01", "Instance State": "running", "Launch Date": "September 10 2019"}
121 return mydict
122
123def connect_db():
124 try:
125 myclient = MongoClient(
126 host = "mongodb://localhost:27017/",
127 serverSelectionTimeoutMS = 3000 # 3 second timeout
128 )
129 except errors.ServerSelectionTimeoutError as e:
130 # set the client instance to 'None' if exception
131 myclient = None
132 # catch pymongo.errors.ServerSelectionTimeoutError
133 print ("pymongo ERROR:", e)
134 return myclient
135
136def set_db():
137 if __name__ == "__main__":
138 message = "* Select a MongoDB Database *"
139 print(Fore.CYAN)
140 banner(message, "*")
141 print(Fore.RESET)
142 myclient = connect_db()
143 today = datetime.today()
144 today = today.strftime("%m%d%Y")
145 if __name__ == "__main__":
146 print(Fore.CYAN + "Available MongoDB Databases:")
147 if myclient != None:
148 # the list_database_names() method returns a list of strings
149 database_names = myclient.list_database_names()
150 counter = 1
151 for db in database_names:
152 message = str(counter) + ". " + db
153 print(message)
154 counter = counter + 1
155 print ("There are", len(database_names), "databases.\n")
156 print(f"Please select a database. Enter a number 1 through {len(database_names)}.")
157 choice = input("Enter a number: ")
158 if is_digit(choice) == True:
159 if int(choice) > counter:
160 print("Wrong selection.")
161 set_db()
162 choice = int(choice)
163 choice = choice - 1
164 mydb = myclient[database_names[choice]]
165 mydb_name = database_names[choice]
166 instance_col = "ec2_list_" + today
167 instance_col = mydb[instance_col]
168 print(f"You've selected: {database_names[choice]}\n")
169 else:
170 print("Must enter a digit. Try again.\n")
171 else:
172 mydb = myclient["aws_inventories"]
173 mydb_name = "aws_inventories"
174 instance_col = "ec2_list_" + today
175 instance_col = mydb[instance_col]
176 return mydb, mydb_name, instance_col
177
178def create_mongodb(mydict):
179 myclient = connect_db()
180 message = f"* Create new MongoDB *"
181 banner(message, border="*")
182 print("\n")
183 newdb = input("Enter the name of a new mongo database: ")
184 dblist = myclient.list_database_names()
185 if newdb in dblist:
186 print("The database exists.")
187 main()
188 else:
189 try:
190 mydb = myclient[newdb]
191 mycol = mydb["testColumn"]
192 mycol.insert_one(mydict)
193 message = f"Succeeded in creating: {newdb}"
194 banner(message)
195 except Exception as e:
196 print(f"MongoDB Database creation failed with: {e}")
197
198def drop_mongodb():
199 message = "* Drop MongoDB *"
200 banner(message, "*")
201 myclient = connect_db()
202 today = datetime.today()
203 today = today.strftime("%m%d%Y")
204 if myclient != None:
205 # the list_database_names() method returns a list of strings
206 database_names = myclient.list_database_names()
207 counter = 1
208 for db in database_names:
209 message = str(counter) + ". " + db
210 print(message)
211 counter = counter + 1
212 print ("There are", len(database_names), "databases.\n")
213 db_names_before_drop = myclient.list_database_names()
214 print ("db count BEFORE drop:", len(db_names_before_drop))
215 print(f"Please select a database. Enter a number 1 through {len(database_names)}.")
216 choice = input("Enter a number: ")
217 if is_digit(choice) == True:
218 if int(choice) > counter:
219 print("Wrong selection.")
220 set_db()
221 choice = int(choice)
222 choice = choice - 1
223 dropdb = myclient[database_names[choice]]
224 instance_col = "ec2_list_" + today
225 instance_col = dropdb[instance_col]
226 print(f"You've selected: {database_names[choice]}\n")
227 else:
228 print("Must enter a digit. Try again.\n")
229 # check if a collection exists
230 col_exists = instance_col in dropdb.list_collection_names()
231 print ("Some Collection exists:", col_exists) # will print True or False
232 # call MongoDB client object"s drop_database() method to delete a db
233 myclient.drop_database(dropdb) # pass db name as string
234 time.sleep(5)
235 # get all of the database names
236 db_names_after_drop = myclient.list_database_names()
237 print ("db count AFTER drop:", len(db_names_before_drop))
238 diff = len(db_names_before_drop) - len(db_names_after_drop)
239 print ("difference:", diff)
240
241def insert_doc(mydict):
242 _, _, instance_col = set_db()
243 mydict["_id"] = ObjectId()
244 instance_doc = instance_col.insert_one(mydict)
245 if __name__ == "__main__":
246 message = "* MongoDB Insert Document *"
247 banner(message, "*")
248 message = f"MongoDB record inserted: {instance_doc.inserted_id}"
249 banner(message)
250 return instance_doc
251
252def mongo_select_all():
253 _, mydb_name, instance_col = set_db()
254 instance_list = list(instance_col.find())
255 if __name__ == "__main__":
256 message = f"* Print DB Documents in {mydb_name} *"
257 banner(message, border="*")
258 print("\n")
259 if not instance_list:
260 message = f"The database: {mydb_name} has no entries."
261 banner(message)
262 else:
263 message = f"The databse: {mydb_name} has {len(instance_list)} entries."
264 for data in instance_list:
265 print(data)
266 print("\n")
267 return instance_list
268
269def mongo_export_to_file(interactive, aws_account, aws_account_number,instance_col=None):
270 create_directories()
271 today = datetime.today()
272 today = today.strftime("%m-%d-%Y")
273 if not instance_col:
274 _, _, instance_col = set_db()
275 time.sleep(10)
276 # make an API call to the MongoDB server
277 if interactive == 0:
278 mongo_docs = instance_col.find()
279 else:
280 mongo_docs = instance_col.find({"Account Number": aws_account_number})
281 # Convert the mongo docs to a DataFrame
282 docs = pandas.DataFrame(mongo_docs)
283 # Discard the Mongo ID for the documents
284 docs.pop("_id")
285 if __name__ == "__main__":
286 print("Choose a file format")
287 print("1. CSV")
288 print("2. JSON")
289 print("3. HTML")
290 print("4. Excel")
291 choice = input("Enter a number 1-4: ")
292 choice = int(choice)
293 else:
294 choice = 1
295 if choice == 1:
296 if __name__ == "__main__":
297 # export MongoDB documents to CSV
298 csv_export = docs.to_csv(sep=",") # CSV delimited by commas
299 print ("\nCSV data:", csv_export)
300 # Set the CSV output directory
301 output_dir = os.path.join("..", "..", "output_files", "aws_instance_list", "csv", "")
302 if interactive == 1:
303 output_file = os.path.join(output_dir, "aws-instance-list-" + aws_account + "-" + today +".csv")
304 else:
305 output_file = os.path.join(output_dir, "aws-instance-master-list-" + today +".csv")
306
307 # export MongoDB documents to a CSV file, leaving out the row "labels" (row numbers)
308 docs.to_csv(output_file, ",", index=False) # CSV delimited by commas
309 elif choice == 2:
310 if __name__ == "__main__":
311 json_export = docs.to_json() # return JSON data
312 print ("\nJSON data:", json_export)
313 # Set the JSON output directory
314 output_dir = os.path.join("..", "..", "output_files", "aws_instance_list", "json", "")
315 if interactive == 1:
316 output_file = os.path.join(output_dir, "aws-instance-list-" + aws_account + "-" + today +".json")
317 else:
318 output_file = os.path.join(output_dir, "aws-instance-master-list" + today +".json")
319 # export MongoDB documents to a CSV file, leaving out the row "labels" (row numbers)
320 docs.to_json(output_file)
321 elif choice == 3:
322 html_str = io.StringIO()
323 # export as HTML
324 docs.to_html(
325 buf=html_str,
326 classes="table table-striped"
327 )
328 if __name__ == "__main__":
329 # print out the HTML table
330 print (html_str.getvalue())
331 # Set the HTML output directory
332 output_dir = os.path.join("..", "..", "output_files", "aws_instance_list", "html", "")
333 if interactive == 1:
334 output_file = os.path.join(output_dir, "aws-instance-list-" + aws_account + "-" + today +".html")
335 else:
336 output_file = os.path.join(output_dir, "aws-instance-master-list" + today + ".html")
337 # save the MongoDB documents as an HTML table
338 docs.to_html(output_file)
339 elif choice == 4:
340 # Set the Excel output directory
341 output_dir = os.path.join("..", "..", "output_files", "aws_instance_list", "excel", "")
342 time.sleep(5)
343 if interactive == 1:
344 output_file = os.path.join(output_dir, "aws-instance-list-" + aws_account + "-" + today + ".xlsx")
345 else:
346 output_file = os.path.join(output_dir, "aws-instance-master-list" + today + ".xlsx")
347 # export MongoDB documents to a Excel file, leaving out the row "labels" (row numbers)
348 writer = ExcelWriter(output_file)
349 docs.to_excel(writer,"EC2 List",index=False)
350 writer.save()
351 if __name__ == "__main__":
352 exit = input("Exit program (y/n): ")
353 if exit.lower() == "y" or exit.lower() == "yes":
354 exit_program()
355
356def clear_db():
357 _, _, instance_col = set_db()
358 message = f"* Clear the DB *"
359 banner(message, border="*")
360 print(f"This command empties the database.\n")
361 try:
362 x = instance_col.delete_many({})
363 except Exception as e:
364 print(f"An error has occurred: {e}")
365 print(x.deleted_count, "documents deleted.")
366
367def delete_from_collection(aws_account_number):
368 _, _, instance_col = set_db()
369 if __name__ == "__main__":
370 message = f"* Clear old entries *"
371 banner(message, border="*")
372 print(f"This command clears old entries the database.\n")
373 aws_account_number = input("Enter an AWS account number: ")
374 try:
375 #instance_col.remove({"Account Number": aws_account_number});
376 instance_col.delete_many({"Account Number": aws_account_number})
377 except Exception as e:
378 print(f"An error has occurred: {e}")
379
380def print_db_names():
381 myclient = connect_db()
382 message = f"* Print DB Names *"
383 banner(message, border="*")
384 print("The database names are:")
385 if myclient != None:
386 # the list_database_names() method returns a list of strings
387 database_names = myclient.list_database_names()
388 counter = 1
389 for db in database_names:
390 message = str(counter) + ". " + db
391 print(message)
392 counter = counter + 1
393 print ("There are", len(database_names), "databases.")
394
395def print_collections():
396 myclient = connect_db()
397 message = f"* Print DB Collections *"
398 banner(message, border="*")
399 print(f"This command prints the database collection names.\n")
400 if myclient != None:
401 # the list_database_names() method returns a list of strings
402 database_names = myclient.list_database_names()
403 print ("There are", len(database_names), "databases.")
404 for db_num, db in enumerate(database_names):
405 print ("\nGetting collections for database:", db, "--", db_num)
406 collection_names = myclient[db].list_collection_names()
407 print ("The MongoDB database returned", len(collection_names), "collections.")
408 # iterate over the list of collection names
409 for col_num, col in enumerate(collection_names):
410 print (col, "--", col_num)
411
412def print_reports(interactive,aws_account,aws_account_number):
413 inputDate = input("Enter the date in format 'dd/mm/yyyy': ")
414 day,month,year = inputDate.split('/')
415 isValidDate = True
416 try:
417 datetime(int(year),int(month),int(day))
418 except ValueError :
419 isValidDate = False
420 print_reports(interactive,aws_account,aws_account_number)
421
422 if(isValidDate) :
423 print(f"Input date is valid: {inputDate}")
424 format= "%m%d%Y"
425 inputDate = datetime.strptime(inputDate,"%m/%d/%Y")
426 inputDate = inputDate.strftime(format)
427 else:
428 print(f"Input date is not valid: {inputDate}")
429 print_reports(interactive,aws_account,aws_account_number)
430 instance_col = "ec2_list_" + inputDate
431 mongo_export_to_file(interactive, aws_account, aws_account_number,instance_col)
432
433
434 #print(f"New date format: {inputDate}")
435 #print_reports()
436
437def menu():
438 message = "Main Menu"
439 banner(message)
440 print(Fore.CYAN + "Your available actions: ")
441 print("1. Create new MongoDB Database")
442 print("2. Drop MongoDB Database")
443 print("3. Do a test insert to the DB")
444 print("4. Clear the DB")
445 print("5. Remove accounts from the DB.")
446 print("6. Print the DB")
447 print("7. Print DB Names")
448 print("8. Print collections")
449 print("9. Export MongoDB to file")
450 print("10. Print Reports")
451 print("11. Exit ec2 mongo")
452 print("\n")
453
454def main():
455 options = arguments()
456 welcomebanner()
457 mydict = set_test_dict()
458 if __name__ == "__main__":
459 print(Fore.YELLOW)
460 aws_accounts_answer = input("Work in one or all accounts: ")
461 print(Fore.RESET)
462 # Set interacive variable to indicate one or many accounts
463 if aws_accounts_answer.lower() == "one" or aws_accounts_answer.lower() == "1":
464 interactive = 1
465 else:
466 interactive = 0
467 aws_account = ''
468 _, aws_env_list = initialize(interactive, aws_account)
469 menu()
470 option = input("Enter the option: ")
471 option = int(option)
472 # 1. Create a MongoDB database
473 if option == 1:
474 create_mongodb(mydict)
475 main()
476 # 2. Drop a MongoDB Database
477 elif option == 2:
478 drop_mongodb()
479 main()
480 # 3. Do a test insert to the DB
481 elif option == 3:
482 insert_doc(mydict)
483 main()
484 # 4. Clear the DB"
485 elif option == 4:
486 clear_db()
487 main()
488 # 5. Remove accounts from the DB.
489 elif option == 5:
490 _, aws_account_number = select_account(options, aws_env_list)
491 delete_from_collection(aws_account_number)
492 main()
493 # 6. Print the DB
494 elif option == 6:
495 mongo_select_all()
496 main()
497 # 7. Print DB Names
498 elif option == 7:
499 print_db_names()
500 main()
501 # 8. Print collections
502 elif option == 8:
503 print_collections()
504 main()
505 # 9. Export MongoDB to file
506 elif option == 9:
507 if aws_accounts_answer == "all":
508 aws_account = "all"
509 aws_account_number = "123456789101"
510 else:
511 aws_account, aws_account_number = select_account(options, aws_env_list)
512 mongo_export_to_file(interactive, aws_account, aws_account_number)
513 main()
514 # 10 Print Reports
515 elif option == 10:
516 if aws_accounts_answer == "all":
517 aws_account = "all"
518 aws_account_number = "123456789101"
519 else:
520 aws_account, aws_account_number = select_account(options, aws_env_list)
521 print_reports(interactive,aws_account,aws_account_number)
522 # 11. Exit ec2 mongo
523 elif option == 11:
524 exit_program()
525 # Invalid Input
526 else:
527 message = "That is not a valid option."
528 banner(message)
529 main()
530
531if __name__ == "__main__":
532 main()