iGNaXBEH

1# Create New Glue Jobs from TextFile Template
2#Assumes Text File has no "" characters (i.e. Name = SLFGlueJob#1)
3
4#import Boto3 AWS APIs, pprint (Beautify Jsons files), defaultdict for creating Dict of TextFile
5import boto3, pprint
6#import awsKeys  #Authentication to Create keys for all environments (Prod, NonProd, Lab, Stage)
7
8from collections import defaultdict
9#assign Glue Api to variable 'Client'
10#client = boto3.client('glue')
11
12#Open the TextFile containing the Glue Job Params, create a dict, read lines, and split dict.keys and dict.values. Re-loop and assign
13# as 'd' update
14filePath = r'C:\Users\gd92\OneDrive - Sun Life Financial\AWS-CANA007WGC\Glue\Cidm Project\JazTest.txt'
15
16
17def gluedictFile():
18    with open(filePath) as gluePromo:
19        d = defaultdict(dict)
20        current = 1
21        file_contents = gluePromo.read()
22        for line in file_contents.splitlines():
23            if "=" in line:
24                key, value = map(str.strip, line.split('='))
25                if key in d[current]:
26                    current += 1
27                d[current][key] = value
28    gluePromo.close()
29    return d
30
31glueDict = gluedictFile()
32pprint.pprint(glueDict)
33#Loop over the key, vals in the dictionary created above. Extract Parameter information and assign below for Glue Job Creation
34for key, value in glueDict.items():
35    glueJobName = glueDict[key]['Name']
36    glueJobDesc = glueDict[key]['Description']
37    glueJobRole = glueDict[key]['Role']
38    glueScript = glueDict[key]['ScriptLocation']
39    glueTempDir = glueDict[key]['TempDir']
40    gluePythonLib = glueDict[key]['PythonLibraryPath']
41    glueEnvDir = glueDict[key]['envFilePath']
42    glueCfgDir = glueDict[key]['cfgFilePath']
43    glueWorkerT = glueDict[key]['WorkerType']
44    glueWorkerCount = glueDict[key]['NumberOfWorkers']
45    #Pass glueWorkCount as int, Boto3 API only accepts INT variable for # of Glue Workers
46    glueWorkersC = int(glueWorkerCount)
47
48print(glueJobName)
49print("Creating new Glue Job ......." + glueJobName)
50    # Initialize Glue Create Job API and feed variables from above Dict into Boto3 Glue Json Args below
51    response = client.create_job(
52        Name=glueJobName,
53        Description=glueJobDesc,
54        Role=glueJobRole,
55
56        Command={
57            'Name': 'glueetl',
58            'ScriptLocation': glueScript,
59            'PythonVersion': '2'
60        },
61        DefaultArguments={
62            '--job-language': 'python',
63            '--enable-metrics': " ",
64            "--TempDir": glueTempDir,
65            '--job-bookmark-option': 'job-bookmark-disable',
66            '--extra-py-files': gluePythonLib,
67            '--env_file_path': glueEnvDir,
68            '--cfg_dir_path': glueCfgDir,
69
70        },
71        #Timeout=3600,
72        #MaxCapacity=10,
73        NumberOfWorkers=glueWorkersC,
74        WorkerType=glueWorkerT,
75
76        )
77    print("Glue Job Created......." + glueJobName)