· 6 years ago · Jul 03, 2019, 02:24 PM
1import requests
2import json
3import pprint
4import os
5import time
6import datetime
7
8def accessdb_fb_auth(dsn, SECRET_KEY, email, admin=False):
9 from firebase import firebase
10 import datetime
11
12 auth = firebase.FirebaseAuthentication(SECRET_KEY, email, admin,
13 admin) # funzione che serve a creare l'autenticazione
14 db = firebase.FirebaseApplication(dsn,
15 auth) # accedo al database con l'url "dsn" e l'autorizzazione e istanzio l'oggetto db
16
17
18 return db # ritorna l'oggetto tramite il quale accediamo al DataBase
19
20def unique(origArr):
21 newArr = []
22 origLen = len(origArr)
23 x_est = 0
24 y_est = 0
25 index = []
26 index2 = []
27 data_incomplete = {}
28 for x in range(origLen):
29 x_est = x
30 found = None
31 for y in range(len(newArr)):
32 y_est = y
33 if (origArr[x] == newArr[y]):
34 found = True
35 break
36 if not found:
37 newArr.append(origArr[x])
38 data_incomplete[origArr[x]] = []
39 else:
40 index.append(x_est)
41 index2.append(y_est)
42 return newArr, index, index2, data_incomplete
43
44
45def get_unique_path_feeds(paths, feeds):
46 # print(feeds)
47 # print(paths)
48 new_arr, index1, index2, data_incomplete = unique(paths)
49 # print("newArr")
50 # pprint.pprint(new_arr)
51 # print("index1")
52 # print(index1)
53 # print("index2")
54 # print(index2)
55 # pprint.pprint(data_incomplete)
56
57 datas = {}
58 data = {}
59
60 for i in range(len(index1)):
61 val = feeds[index1[i]]
62 path = new_arr[index2[i]]
63 data_index = []
64 data_index = data_incomplete[path]
65
66 data_index.append(val)
67 data_incomplete[path] = data_index
68
69 pass
70
71 for x in new_arr:
72 i = 0
73 while True:
74 if (paths[i] == x):
75 val = feeds[i]
76 data_index = data_incomplete[x]
77 data_index.append(val)
78 data_incomplete[x] = data_index
79 break
80 i = i + 1
81
82 return data_incomplete
83
84def get_data_from_database():
85 print("Sort Feed By zone: ")
86 url = 'https://us-central1-sensesquaredb.cloudfunctions.net/sortFeedByZone'
87 response = requests.get(url)
88 print(response)
89
90 date_time = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
91 nomefile = 'data' + date_time + '.json'
92
93 # with open(nomefile,"w") as file:
94 # json.dump(response.json(),file,ensure_ascii=False)
95
96 data = response.json()
97
98 feeds = data['feeds']
99 paths = data['paths']
100
101 data_incomplete = get_unique_path_feeds(paths, feeds)
102
103 result_file = "result_" + nomefile
104
105 # with open(result_file,"w") as file:
106 # json.dump(data_incomplete,file,ensure_ascii=False)
107 #
108 # db=accessdb_fb_auth('https://sensesquaredb.firebaseio.com/','0JuzXIx0GkGP1OKS4LltwLVgQrnaaYX5MDP421KT',"sort_function@gmail.com",True)
109 # db.post
110 return data_incomplete
111
112def update_av(area):
113 url_av_M = 'http://us-central1-sensesquaredb.cloudfunctions.net/update_averageM'
114 url_av_D = 'http://us-central1-sensesquaredb.cloudfunctions.net/update_averageD'
115 url_av_H = 'http://us-central1-sensesquaredb.cloudfunctions.net/update_averageH'
116
117 cap = area['cap']
118 zone = area['zone']
119 responses=[]
120
121 #averageM
122 data = {"path": "Areas/" + cap + "/" + zone , "minute_keys": json.dumps(area['minute_keys'])}
123 #print(data)
124 response = requests.post(url_av_M, data=data)
125 responses.append(response)
126 #print(response.text)
127
128 #averageH
129 data = {"path": "Areas/" + cap + "/" + zone, "hour_keys": json.dumps(area['hour_keys'])}
130 #print(data)
131 response = requests.post(url_av_H, data=data)
132 responses.append(response)
133 #print(response.text)
134
135 #averageD
136 data = {"path": "Areas/" + cap + "/" + zone, "day_keys": json.dumps(area['day_keys'])}
137 #print(data)
138 response = requests.post(url_av_D, data=data)
139 responses.append(response)
140 #print(response.text)
141 print(responses)
142
143def update_av_km(area):
144 url_av_M = 'http://us-central1-sensesquaredb.cloudfunctions.net/update_averageM'
145 url_av_D = 'http://us-central1-sensesquaredb.cloudfunctions.net/update_averageD'
146 url_av_H = 'http://us-central1-sensesquaredb.cloudfunctions.net/update_averageH'
147 responses=[]
148
149 cap = area['cap']
150 zone = area['zone']
151 square=area['square']
152
153 data = {"path": "AreasKM/" + cap + "/" + zone + "/"+str(square), "minute_keys": json.dumps(area['minute_keys'])}
154 #print(data)
155 response = requests.post(url_av_M, data=data)
156 responses.append(response)
157 #print(response.text)
158
159 # averageH
160 data = {"path": "AreasKM/" + cap + "/" + zone + "/"+str(square), "hour_keys": json.dumps(area['hour_keys'])}
161 #print(data)
162 response = requests.post(url_av_H, data=data)
163 responses.append(response)
164 #print(response.text)
165
166 # averageD
167 data = {"path": "AreasKM/" + cap + "/" + zone + "/"+str(square), "day_keys": json.dumps(area['day_keys'])}
168 #print(data)
169 response = requests.post(url_av_D, data=data)
170 responses.append(response)
171 #print(response.text)
172
173 print(responses)
174
175def calcola_unsorted_feeds():
176 db = accessdb_fb_auth('https://sensesquaredb.firebaseio.com/', '0JuzXIx0GkGP1OKS4LltwLVgQrnaaYX5MDP421KT',
177 "sort_function@gmail.com", True)
178 unsorted_feeds = db.get('UnsortedFeeds', '')
179 count = 0
180 for x in unsorted_feeds:
181 count += 1
182 #print("Numero record: " + str(count))
183 return count
184
185def main():
186
187
188 data_incomplete=get_data_from_database()
189 data = {'path_field': json.dumps(data_incomplete)}
190
191 #-------------------------------------------------------
192 # with open('data.json', 'r+') as file:
193 # data = {'path_field': json.dumps(json.load(file))}
194 #------------------------------------------------------
195
196 # pprint.pprint(data)
197 print("Sorted Feed Reorganize")
198 url = 'https://us-central1-sensesquaredb.cloudfunctions.net/sortedFeedReorganize'
199 response = requests.post(url, data=data)
200
201 print("Response")
202 print(response)
203
204 print("Waiting...")
205 time.sleep(60)
206 print("OK")
207
208 db = accessdb_fb_auth('https://sensesquaredb.firebaseio.com/', '0JuzXIx0GkGP1OKS4LltwLVgQrnaaYX5MDP421KT',
209 "sort_function@gmail.com", True)
210
211 support_average = db.get('', 'SupportAverage')
212
213 areas = support_average['Areas']
214 areasKM = support_average['AreasKM']
215
216 db.delete('','SupportAverage')
217
218 areas_to_update = []
219 areasKM_to_update= []
220
221 for cap in areas:
222 # print(cap)
223 for zone in areas[cap]:
224 # print(zone)
225 minute_keys = []
226 hour_keys = []
227 day_keys = []
228 for feed_id in areas[cap][zone]:
229 min_hour_day_month_year = areas[cap][zone][feed_id]
230 hour_day_month_year = min_hour_day_month_year[3:]
231 day_month_year = hour_day_month_year[3:]
232
233 minute_keys.append(min_hour_day_month_year)
234 hour_keys.append(hour_day_month_year)
235 day_keys.append(day_month_year)
236 # print("\n______________________________________")
237 # print(minute_keys)
238 # print(hour_keys)
239 # print(day_keys)
240
241 minute_keys = unique(minute_keys)[0]
242 hour_keys = unique(hour_keys)[0]
243 day_keys = unique(day_keys)[0]
244 # print("- - - - - - - - - - - - - - - - - - - - ")
245 # print(minute_keys)
246 # print(hour_keys)
247 # print(day_keys)
248 # print("______________________________________\n")
249
250 areas_to_update.append({"cap": cap, "zone": zone, "minute_keys": minute_keys,
251 "hour_keys": hour_keys, "day_keys": day_keys
252 })
253
254 for cap in areasKM:
255 #print(cap)
256 for zone in areasKM[cap]:
257 #print(zone)
258 i = 0
259 for square in areasKM[cap][zone]:
260
261 if square:
262 #print(square)
263 minute_keys = []
264 hour_keys = []
265 day_keys = []
266 if type(square) is dict:
267 for feed_id in square:
268 min_hour_day_month_year = square[feed_id]
269 hour_day_month_year = min_hour_day_month_year[3:]
270 day_month_year = hour_day_month_year[3:]
271
272 minute_keys.append(min_hour_day_month_year)
273 hour_keys.append(hour_day_month_year)
274 day_keys.append(day_month_year)
275 square=i
276
277 else:
278 for feed_id in areasKM[cap][zone][square]:
279 min_hour_day_month_year = areasKM[cap][zone][square][feed_id]
280 hour_day_month_year = min_hour_day_month_year[3:]
281 day_month_year = hour_day_month_year[3:]
282
283 minute_keys.append(min_hour_day_month_year)
284 hour_keys.append(hour_day_month_year)
285 day_keys.append(day_month_year)
286
287
288 # print("\n______________________________________")
289 # print(minute_keys)
290 # print(hour_keys)
291 # print(day_keys)
292
293 minute_keys = unique(minute_keys)[0]
294 hour_keys = unique(hour_keys)[0]
295 day_keys = unique(day_keys)[0]
296 # print("- - - - - - - - - - - - - - - - - - - - ")
297 # print(minute_keys)
298 # print(hour_keys)
299 # print(day_keys)
300 # print("______________________________________\n")
301
302 areasKM_to_update.append(
303 {'cap': cap, 'zone': zone, 'square': str(square), 'minute_keys': minute_keys, 'hour_keys': hour_keys,
304 'day_keys': day_keys})
305 i+=1
306
307 print("---------------------------------------")
308 #print("areas to update")
309 #print(areas_to_update)
310 print("---------------------------------------")
311 #print("areasKMtoupdate")
312 #print(json.dumps(areasKM_to_update))
313
314 time.sleep(5)
315
316 for area in areas_to_update:
317 print(area)
318 update_av(area)
319
320 for area in areasKM_to_update:
321 print(area)
322 update_av_km(area)
323
324
325if __name__=='__main__':
326
327 try:
328 date_start=datetime.datetime.now()
329 count=calcola_unsorted_feeds()
330
331 print("start")
332 main()
333
334 date_finish=datetime.datetime.now()
335 delta=date_finish-date_start
336 seconds=round(1200-delta.total_seconds())
337 print("Numero record: " + str(count))
338 print("Il programma ha impiegato "+str(delta.total_seconds())+" secondi.")
339
340 except Exception as error:
341 timestamp=datetime.datetime.now().strftime("%d/%m/%Y %H:%M,%S")
342 with open("errorlog.txt","a+") as file:
343 file.write("\n----------------------------")
344 file.write(timestamp)
345 file.write(str(error))