C1w1YKLS

1# import config.config as config
2
3
4ACCESS_KEY = "" #config.aws_keys['access_key_id']
5SECRET_KEY = "" #config.aws_keys['secret_access_key']
6BUCKET_NAME = "polysemy-embedding-sutha" #config.bucket_name
7JOB_NAME=""
8Iteration="iteration-2"
9dataFile = "corpus"
10
11import boto3
12
13import boto3
14import os
15
16
17def upload_files(folder_path,upload_path):
18    s3 = boto3.client('s3')
19    s3.create_bucket(Bucket=BUCKET_NAME)
20    response = s3.list_buckets()
21
22    # Get a list of all bucket names from the response
23    buckets = [bucket['Name'] for bucket in response['Buckets']]
24
25    # Print out the bucket list
26    print("Bucket List: %s" % buckets)
27    s3.upload_file(folder_path, BUCKET_NAME, upload_path, ExtraArgs={'ACL':'public-read'})
28
29
30
31import re
32# from sagemaker import get_execution_role
33import boto3
34from time import gmtime, strftime
35import time
36import numpy as np
37import os
38import json
39from validate import validate_params
40import logging
41from datetime import datetime
42
43logging.getLogger('botocore').setLevel(logging.DEBUG)
44logging.getLogger('boto3').setLevel(logging.DEBUG)
45
46
47def embed(bucket,prefix,output):
48    # role=get_execution_role()
49    role = "arn:aws:iam::779672335440:role/sageMakerFullAccess"
50    print(role)
51    region_name = boto3.Session().region_name
52    print(region_name)
53
54    containers = {'us-west-2': '433757028032.dkr.ecr.us-west-2.amazonaws.com/blazingtext:latest',
55                  'us-east-1': '811284229777.dkr.ecr.us-east-1.amazonaws.com/blazingtext:latest',
56                  'us-east-2': '825641698319.dkr.ecr.us-east-2.amazonaws.com/blazingtext:latest',
57                  'eu-west-1': '685385470294.dkr.ecr.eu-west-1.amazonaws.com/blazingtext:latest'}
58
59    container = containers[region_name]
60    print('Using SageMaker BlazingText container: {} ({})'.format(container, region_name))
61
62    resource_config = {
63            "InstanceCount": 1,
64            "InstanceType": "ml.c4.8xlarge",
65            "VolumeSizeInGB": 50
66        }
67
68    epochCount = 5
69    hyperparameters = {
70            "mode": "skipgram",
71            "epochs": str(epochCount),
72            "min_count": "4",
73            "sampling_threshold": "0.0001",
74            "learning_rate": "0.04",
75            "window_size": "5",
76            "vector_dim": "100",
77            "negative_samples": "5",
78            "batch_size": "11", #  = (2*window_size + 1) (Preferred)
79            "evaluation": "true" # Perform similarity evaluation on WS-353 dataset at the end of training
80        }
81
82
83    validate_params(resource_config, hyperparameters)
84
85
86    job_name = "Embedding-{}-{}-{}-{}-".format(str(epochCount),resource_config["InstanceCount"],
87                                                resource_config["InstanceType"].replace(".","-"),
88                                                hyperparameters["mode"].replace("_","-"))\
89                                        + strftime("%Y-%m-%d-%H-%M", gmtime())
90    job_name += "-" + dataFile
91
92    global JOB_NAME
93    JOB_NAME= job_name
94    print("Training job", job_name)
95
96
97    create_training_params = \
98    {
99        "TrainingJobName": job_name,
100        "ResourceConfig": resource_config,
101        "HyperParameters": hyperparameters,
102        "AlgorithmSpecification": {
103            "TrainingImage": container,
104            "TrainingInputMode": "File"
105        },
106        "RoleArn": role,
107        "OutputDataConfig": {
108            "S3OutputPath": "s3://{}/{}/".format(bucket, output)
109        },
110        "StoppingCondition": {
111            "MaxRuntimeInSeconds": 3600 * 24 #Hours
112        },
113        "InputDataConfig": [
114            {
115                "ChannelName": "train",
116                "DataSource": {
117                    "S3DataSource": {
118                        "S3DataType": "S3Prefix",
119                        "S3Uri": "s3://{}/{}/".format(bucket, prefix),
120                        "S3DataDistributionType": "FullyReplicated"  # Always keep FullyReplicated for BlazingText
121                    }
122                },
123            },
124        ]
125    }
126    startTime = datetime.now()
127    sagemaker_client = boto3.Session().client(service_name='sagemaker')
128    sagemaker_client.create_training_job(**create_training_params)
129    status = sagemaker_client.describe_training_job(TrainingJobName=job_name)['TrainingJobStatus']
130    print(status)
131
132    time.sleep(5)
133    sagemaker_client.get_waiter('training_job_completed_or_stopped').wait(TrainingJobName=job_name)
134
135    status = sagemaker_client.describe_training_job(TrainingJobName=job_name)['TrainingJobStatus']
136    print(status)
137
138    # if the job failed, determine why
139    if status == 'Failed':
140        message = sagemaker_client.describe_training_job(TrainingJobName=job_name)['FailureReason']
141        print('Training failed with the following error: {}'.format(message))
142        raise Exception('Training job failed')
143
144
145    endtime=datetime.now()
146    print("Duration : ",(endtime-startTime))
147
148import botocore
149def download(folder_path,download_path):
150
151    s3 = boto3.resource('s3')
152    try:
153        s3.Bucket(BUCKET_NAME).download_file(folder_path, download_path)
154    except botocore.exceptions.ClientError as e:
155        if e.response['Error']['Code'] == "404":
156            print("The object does not exist.")
157        else:
158            raise
159
160
161
162
163if __name__ == "__main__":
164    # upload_files("/home/piraveena/Documents/Semi8/fyp-polysemy-embedding/data/histogram/text8",Iteration+"/file/data.txt")
165    embed(BUCKET_NAME,Iteration+"/"+dataFile,Iteration+"/"+dataFile+"_out")
166    #download(Iteration+"/model/"+JOB_NAME+"/output/model.tar.gz","/home/piraveena/Documents/Semi8/fyp-polysemy-embedding/data/histogram/we_model.tar")
167    #