JWxhaw9E

1# set working directory (location where code is)
2import os
3os.chdir("path")
4 
5import re
6import csv
7import time
8import pandas as pd
9from random import randint
10from GT_Automation_Code import pyGTrends
11 
12# set gmail credentials and path to extract data
13google_username = "*****@gmail.com"
14google_password = "*****"
15 
16Daily_Data = [ ]
17         
18# define daily pull code
19def GT_Daily_Run(keys):
20     
21    path = 'path'
22 
23    # connect to Google
24    connector = pyGTrends(google_username, google_password)
25    # make request
26    connector.request_report(keys, date="today 90-d", geo="US")
27    # wait a random amount of time between requests to avoid bot detection
28    time.sleep(randint(5, 10))
29    # download file
30    connector.save_csv(path, '_' + "GT_Daily" + '_' + keys.replace(' ', '_'))
31 
32    name = path + '_' + "GT_Daily" + '_' + keys.replace(' ', '_')
33       
34    with open(name + '.csv', 'rt') as csvfile:    
35        csvReader = csv.reader(csvfile)
36        data = []
37 
38        for row in csvReader:
39            if any('2015' in s for s in row): 
40                data.append(row)
41 
42        day_df = pd.DataFrame(data)
43        cols = ["Day", keys]    
44        day_df.columns = [cols]  
45        Daily_Data.append(day_df) 
46 
47keywords = ['soccer', 'football', 'baseball']    
48 
49map(lambda x: GT_Daily_Run(x), keywords)  
50 
51rge = [Daily_Data[0], Daily_Data[1], Daily_Data[2]]    
52 
53df_final_daily = reduce(lambda left,right: pd.merge(left,right, on='Day'), rge)
54df_final_daily = df_final_daily.loc[:, (df_final_daily != "0").any(axis=0)]
55df_final_daily.to_csv("Daily_Trends_Data.csv", index=False)
56 
57Weekly_Data = [ ]     
58         
59# define weekly pull code
60def GT_Weekly_Run(keys):
61 
62    path = 'path'
63     
64    # connect to Google
65    connector = pyGTrends(google_username, google_password)
66    # make request
67    connector.request_report(keys, geo="US")
68    # wait a random amount of time between requests to avoid bot detection
69    time.sleep(randint(5, 10))
70    # download file
71    connector.save_csv(path, '_' + "GT_Weekly" + '_' + keys.replace(' ', '_'))
72 
73    name = path + '_' + "GT_Weekly" + '_' + keys.replace(' ', '_') 
74 
75    with open(name + '.csv', 'rt') as csvfile:    
76        csvReader = csv.reader(csvfile)
77        data = []
78        datex = re.compile('(19|20)dd-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])')
79 
80        for row in csvReader:
81            if datex.search(str(row)):
82                data.append(row)
83        
84        week_df = pd.DataFrame(data)
85        cols = ["Week", keys]    
86        week_df.columns = [cols]
87        Weekly_Data.append(week_df)              
88              
89map(lambda x: GT_Weekly_Run(x), keywords)  
90 
91rge = [Weekly_Data[0], Weekly_Data[1], Weekly_Data[2]]    
92 
93df_final_weekly = reduce(lambda left,right: pd.merge(left,right, on='Week'), rge)
94df_final_weekly = df_final_weekly.loc[:, (df_final_weekly != "0").any(axis=0)] 
95df_final_weekly.to_csv("Weekly_Trends_Data.csv", index=False)