· 6 years ago · May 14, 2019, 10:46 AM
1Client.py
2
3import requests
4import time
5import json
6
7
8def printRequestInfo(r):
9 print('request.url:' + str(r.url))
10 print('request.status_code:' + str(r.status_code))
11 print('request.headers: ' + str(r.headers))
12 print('request.text: ' + str(r.text))
13 print('request.request.body: ' + str(r.request.body))
14 print('request.request.header: ' + str(r.request.headers))
15 print('====================================================')
16 print('\n')
17 time.sleep(0.1)
18
19port = '8080'
20
21def getRatings():
22 r = requests.get('http://localhost:'+port+'/ratings')
23 printRequestInfo(r)
24
25
26def addRating(data):
27 headers = {'content-type': 'application/json'}
28 r = requests.post('http://localhost:'+port+'/rating', data=json.dumps(data), headers=headers)
29 printRequestInfo(r)
30
31
32def deleteRatings():
33 r = requests.delete('http://localhost:'+port+'/ratings')
34 printRequestInfo(r)
35
36
37def getAvgGenresAllUsers():
38 r = requests.get('http://localhost:'+port+'/avg-genre-ratings/all-users')
39 printRequestInfo(r)
40
41
42def getAvgGenresByUser(user_id):
43 r = requests.get('http://localhost:'+port+'/avg-genre-ratings/' + user_id)
44 printRequestInfo(r)
45
46def getUserProfile(user_id):
47 r = requests.get('http://localhost:' + port + '/profile/' + user_id)
48 printRequestInfo(r)
49
50
51
52if __name__ == '__main__':
53 data = {"userID": 78, "movieID": 904, "rating": 3.0, "genre-Action": 0, "genre-Adventure": 0, "genre-Animation": 0,
54 "genre-Children": 0, "genre-Comedy": 0, "genre-Crime": 0, "genre-Documentary": 0, "genre-Drama": 1,
55 "genre-Fantasy": 0,
56 "genre-Film-Noir": 0, "genre-Horror": 0, "genre-IMAX": 0, "genre-Musical": 0, "genre-Mystery": 1,
57 "genre-Romance": 1,
58 "genre-Sci-Fi": 0, "genre-Short": 0, "genre-Thriller": 0, "genre-War": 0, "genre-Western": 0}
59 addRating(data)
60 data = {"userID": 78, "movieID": 903, "rating": 3.0, "genre-Action": 0, "genre-Adventure": 0, "genre-Animation": 0,
61 "genre-Children": 0, "genre-Comedy": 0, "genre-Crime": 0, "genre-Documentary": 0, "genre-Drama": 1,
62 "genre-Fantasy": 0,
63 "genre-Film-Noir": 0, "genre-Horror": 0, "genre-IMAX": 0, "genre-Musical": 0, "genre-Mystery": 1,
64 "genre-Romance": 1,
65 "genre-Sci-Fi": 0, "genre-Short": 0, "genre-Thriller": 0, "genre-War": 0, "genre-Western": 0}
66 addRating(data)
67 data = {"userID": 79, "movieID": 904, "rating": 2.0, "genre-Action": 0, "genre-Adventure": 0, "genre-Animation": 0,
68 "genre-Children": 0, "genre-Comedy": 0, "genre-Crime": 0, "genre-Documentary": 0, "genre-Drama": 1,
69 "genre-Fantasy": 0,
70 "genre-Film-Noir": 0, "genre-Horror": 0, "genre-IMAX": 0, "genre-Musical": 0, "genre-Mystery": 1,
71 "genre-Romance": 1,
72 "genre-Sci-Fi": 0, "genre-Short": 0, "genre-Thriller": 1, "genre-War": 0, "genre-Western": 0}
73 addRating(data)
74 getRatings()
75 getAvgGenresAllUsers()
76 getAvgGenresByUser('78')
77 getUserProfile('78')
78
79CassandraClient.py
80
81from cassandra.cluster import Cluster
82from cassandra.query import dict_factory
83import Lab4
84import json
85
86KEYSPACE = "rating_keyspace"
87USER_TABLE = "user_rated_movies"
88AVG_GENRE_RATINGS_FOR_USER = "avg_genre_ratings_for_user"
89
90class CassandraCli:
91
92
93 def __init__(self):
94 cluster = Cluster(['127.0.0.1'], port=9042)
95 self.session = cluster.connect()
96 self.create_keyspace()
97 self.session.set_keyspace(KEYSPACE)
98 self.session.row_factory = dict_factory
99
100 def create_keyspace(self):
101 self.session.execute("""CREATE KEYSPACE IF NOT EXISTS """+KEYSPACE+"""
102 WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': '1' }
103 """)
104
105 def create_table_query(self, keyspace, table, primaryKey, additionalColumns):
106 output = """CREATE TABLE IF NOT EXISTS """ + keyspace + """.""" + table + """ ( """
107 for column_name, type in additionalColumns.items():
108 output = output + str(column_name) + ' ' + str(type) + ', '
109 output = output + 'PRIMARY KEY (' + primaryKey + ')'
110 output = output + \
111 """ );"""
112 return output
113
114 def create_table_query2(self, keyspace, table, primaryKey, secondaryKey, additionalColumns):
115 output = """CREATE TABLE IF NOT EXISTS """ + keyspace + """.""" + table + """ ( """
116 for column_name, type in additionalColumns.items():
117 output = output + str(column_name) + ' ' + str(type) + ', '
118 output = output + 'PRIMARY KEY (' + primaryKey + ',' + secondaryKey + ')'
119 output = output + \
120 """ );"""
121 return output
122
123 def query_helper(self, tableName):
124 _, genres_column_names = Lab4.merge(1)
125 genres_num = len(genres_column_names)
126 genres_column_names = [genre.replace('-', '_') for genre in genres_column_names]
127 if USER_TABLE == tableName:
128 genres_column_names.insert(0, "rating")
129 genres_column_names.insert(0, "movieID")
130 genres_column_names.insert(0, "userID")
131 types = ['int', 'int', 'float']
132 for _ in range(genres_num):
133 types.append('int')
134 else:
135 genres_column_names.insert(0, "userID")
136 types = ['int']
137 for _ in range (genres_num):
138 types.append('float')
139 return dict(zip(genres_column_names, types))
140
141 def create_avg_genre_ratings_for_user_table(self):
142 query = self.create_table_query(KEYSPACE, AVG_GENRE_RATINGS_FOR_USER, 'userID', self.query_helper(AVG_GENRE_RATINGS_FOR_USER))
143 self.session.execute(query)
144
145 def create_avg_genre_ratings_all(self):
146 query = self.create_table_query(KEYSPACE, AVG_GENRE_RATINGS_FOR_USER, 'userID', self.query_helper(AVG_GENRE_RATINGS_FOR_USER))
147 self.session.execute(query)
148
149 def create_user_rated_movies(self):
150 query = self.create_table_query(KEYSPACE, USER_TABLE, 'userID', self.query_helper(USER_TABLE))
151 self.session.execute(query)
152
153 def create_user_rated_movies2(self):
154 query = self.create_table_query2(KEYSPACE, USER_TABLE, 'userID', 'movieID', self.query_helper(USER_TABLE))
155 self.session.execute(query)
156
157 def push_rating(self, row):
158 row = { k.replace('-', '_'): v for k, v in row.items() }
159 row = json.dumps(row)
160 query = "INSERT INTO " + KEYSPACE + "." + USER_TABLE + " JSON '" + row + "';"
161 self.session.execute(query)
162
163
164 def push_avg_ratings(self, row):
165 query = "INSERT INTO " + KEYSPACE + "." + AVG_GENRE_RATINGS_FOR_USER + " JSON '" + row + "';"
166 self.session.execute(query)
167
168
169 def get_avg_user(self, user_id):
170 rows = self.session.execute("SELECT * FROM " + KEYSPACE + "." + AVG_GENRE_RATINGS_FOR_USER + " WHERE userID=" + str(user_id) + " ;")
171 return rows
172
173 def get_ratings(self):
174 rows = self.session.execute(
175 "SELECT * FROM " + KEYSPACE + "." + USER_TABLE + ";")
176 return rows
177
178 def clear_table(self, keyspace, table):
179 self.session.execute("TRUNCATE "+keyspace+"."+table+";")
180
181 def delete_table(self, keyspace, table):
182 self.session.execute("DROP TABLE "+keyspace+"."+table+";")
183
184 #if __name__ == "__main__":
185 # keyspace = "user_ratings"
186 # table = "user_avg_rating"
187 # cluster = Cluster(['127.0.0.1'], port=9042)
188 # session = cluster.connect()
189 # create_keyspace(session, keyspace)
190 # session.set_keyspace(keyspace)
191 # session.row_factory = dict_factory
192 # create_table(session, keyspace, table)
193 #
194 # push_data_table(session, keyspace, table, userId=1337, avgMovieRating=4.2)
195 # get_data_table(session, keyspace, table)
196 # clear_table(session, keyspace, table)
197 # get_data_table(session, keyspace, table)
198 # delete_table(session, keyspace, table)
199
200CherryPy.py
201
202import cherrypy
203import Lab5
204import Lab4
205import json
206import routes
207
208
209
210class Ratings(object):
211 def __init__(self):
212 client_side_df, genres_column_names = Lab4.merge(1)
213 global logic
214 logic = Lab5.api_logic(genres_column_names)
215
216 @cherrypy.expose
217 def get_ratings(self):
218 ratings = logic.list_ratings()[:100]
219 return json.dumps(ratings)
220
221 @cherrypy.expose
222 def delete_ratings(self):
223 logic.delete_ratings()
224 return 'Deleted'
225
226 @cherrypy.expose
227 @cherrypy.tools.accept(media='application/json')
228 def add_rating(self):
229 rawData = cherrypy.request.body.read(int(cherrypy.request.headers['Content-Length']))
230 aNewRating = json.loads(rawData)
231 logic.add_rating(aNewRating)
232 return 'OK'
233
234
235
236class Average(object):
237 @cherrypy.expose
238 def user(self, id):
239 return str(logic.compute_avg_genre_ratings_for_user(int(id)))
240
241 @cherrypy.expose
242 def all(self):
243 return str(logic.compute_avg_genre_ratings())
244
245 @cherrypy.expose
246 def profile(self, id):
247 user_profile = logic.compute_user_profile(id)
248 return json.dumps(list(user_profile))
249
250
251average = Average()
252ratings = Ratings()
253d = cherrypy.dispatch.RoutesDispatcher()
254
255d.connect('ratings', route='/ratings', controller=ratings, action='get_ratings', conditions=dict(method=['GET']))
256d.connect('rating', route='/rating', controller=ratings, action='add_rating', conditions=dict(method=['POST']))
257d.connect('ratings', route='/ratings', controller=ratings, action='delete_ratings', conditions=dict(method=['DELETE']))
258
259d.connect('all', route='/avg-genre-ratings/all-users', controller=average, action='all')
260d.connect('user', route='/avg-genre-ratings/:id', controller=average, action='user')
261d.connect('profile', route='/profile/:id', controller=average, action='profile')
262
263conf = {'/': {'request.dispatch': d}}
264
265if __name__ == '__main__':
266 cherrypy.quickstart(ratings, '/', conf)
267
268Lab5.py
269
270
271import Lab4
272import copy
273import CassandraClient
274from CassandraClient import CassandraCli
275import json
276
277class api_logic:
278
279 def __init__(self, genres_column_names):
280 self.rawRatingsData = []
281 self.genres_column_names = genres_column_names
282 self.avg_genre_ratings_for_user = {}
283 self.user_profile = {}
284 self.cassandraCli = CassandraCli()
285 self.cassandraCli.create_keyspace()
286 self.cassandraCli.create_avg_genre_ratings_for_user_table()
287 self.cassandraCli.create_user_rated_movies2()
288
289 def add_rating(self, aNewRating):
290 self.cassandraCli.push_rating(aNewRating)
291
292 def list_ratings(self):
293 self.rawRatingsData = self.cassandraCli.get_ratings()
294 return self.rawRatingsData
295
296 def delete_ratings(self):
297 self.rawRatingsData = []
298 self.cassandraCli.clear_table(keyspace=CassandraClient.KEYSPACE, table=CassandraClient.USER_TABLE)
299 # tod
300
301 def compute_avg_genre_ratings(self):
302 ratings = self.cassandraCli.get_ratings()
303 r = []
304 for rating in ratings.current_rows:
305 rating = transform_names(rating)
306 r.append(rating)
307 self.DFUserRatedMoviesWithMovieGenres = Lab4.dict_to_DataFrame(copy.deepcopy(r))
308 _ , average_ratings_of_genres = Lab4.get_mean_all_users(self.DFUserRatedMoviesWithMovieGenres, self.genres_column_names)
309 self.avg_genre_ratings = copy.deepcopy(average_ratings_of_genres)
310
311 #self.redis_profiles_client.set("avg_genre_ratings", json.dumps(list(self.avg_genre_ratings)))
312
313
314 return self.avg_genre_ratings
315
316 def compute_avg_genre_ratings_for_user(self, user_id):
317 ratings = self.cassandraCli.get_ratings()
318 r = []
319 for rating in ratings.current_rows:
320 rating = transform_names(rating)
321 r.append(rating)
322
323 self.DFUserRatedMoviesWithMovieGenres = Lab4.dict_to_DataFrame(copy.deepcopy(r))
324 ave = Lab4.get_mean_user(self.DFUserRatedMoviesWithMovieGenres, self.genres_column_names, user_id)
325 self.avg_genre_ratings_for_user[user_id] = ave
326 l = list(ave)
327 dict = {}
328 dict['userID'] = user_id
329 ctr = 0
330 for genre in self.genres_column_names:
331 dict[genre] = l[ctr]
332 dict = {k.replace('-', '_'): v for k, v in dict.items()}
333
334 self.cassandraCli.push_avg_ratings(json.dumps(dict))
335 return self.avg_genre_ratings_for_user[user_id]
336
337 def compute_user_profile(self, user_id):
338 user_id = int(user_id)
339 self.compute_avg_genre_ratings()
340 self.compute_avg_genre_ratings_for_user(user_id)
341 # self.avg_genre_ratings_for_user[user_id] = np.array(json.loads(self.redis_profiles_client.get("avg_genre_ratings_for_user-"+str(user_id))))
342 # self.avg_genre_ratings = np.array(json.loads(self.redis_profiles_client.get("avg_genre_ratings")))
343 self.user_profile[user_id] = list(self.avg_genre_ratings_for_user[user_id] - self.avg_genre_ratings)
344 return self.user_profile[user_id]
345
346
347def transform_names(ratings):
348 ratings = {k.replace('_', '-'): v for k, v in ratings.items()}
349 ratings = {k.replace('genre-action', 'genre-Action'): v for k, v in ratings.items()}
350 ratings = {k.replace('genre-adventure', 'genre-Adventure'): v for k, v in ratings.items()}
351 ratings = {k.replace('genre-animation', 'genre-Animation'): v for k, v in ratings.items()}
352 ratings = {k.replace('genre-children', 'genre-Children'): v for k, v in ratings.items()}
353 ratings = {k.replace('genre-comedy', 'genre-Comedy'): v for k, v in ratings.items()}
354 ratings = {k.replace('genre-crime', 'genre-Crime'): v for k, v in ratings.items()}
355 ratings = {k.replace('genre-documentary', 'genre-Documentary'): v for k, v in ratings.items()}
356 ratings = {k.replace('genre-drama', 'genre-Drama'): v for k, v in ratings.items()}
357 ratings = {k.replace('genre-fantasy', 'genre-Fantasy'): v for k, v in ratings.items()}
358 ratings = {k.replace('genre-film-noir', 'genre-Film-Noir'): v for k, v in ratings.items()}
359 ratings = {k.replace('genre-horror', 'genre-Horror'): v for k, v in ratings.items()}
360 ratings = {k.replace('genre-imax', 'genre-IMAX'): v for k, v in ratings.items()}
361 ratings = {k.replace('genre-musical', 'genre-Musical'): v for k, v in ratings.items()}
362 ratings = {k.replace('genre-mystery', 'genre-Mystery'): v for k, v in ratings.items()}
363 ratings = {k.replace('genre-romance', 'genre-Romance'): v for k, v in ratings.items()}
364 ratings = {k.replace('genre-sci-fi', 'genre-Sci-Fi'): v for k, v in ratings.items()}
365 ratings = {k.replace('genre-short', 'genre-Short'): v for k, v in ratings.items()}
366 ratings = {k.replace('genre-thriller', 'genre-Thriller'): v for k, v in ratings.items()}
367 ratings = {k.replace('genre-war', 'genre-War'): v for k, v in ratings.items()}
368 ratings = {k.replace('genre-western', 'genre-Western'): v for k, v in ratings.items()}
369 ratings = {k.replace('userid', 'userID'): v for k, v in ratings.items()}
370 ratings = {k.replace('movieid', 'movieID'): v for k, v in ratings.items()}
371 return ratings
372
373if __name__ == "__main__":
374 list_of_keys_of_non_lints = ["rating"]
375 #client_side_df, genres_column_names = Lab4.merge(100)
376 #client_side_df['userID'] = client_side_df['userID'].astype('int', copy=False)
377 #api_logic = api_logic(genres_column_names)
378 #row_iterator = client_side_df.iterrows()
379 #for row in row_iterator:
380 # row_as_dict=row[1].to_dict()
381 # api_logic.add_rating(row_as_dict)
382 #list_of_ratings = api_logic.list_ratings()
383 #avg_genre_ratings = api_logic.compute_avg_genre_ratings()
384 #some_user_ID = 75
385 #avg_genre_ratings_for_user = api_logic.compute_avg_genre_ratings_for_user(some_user_ID)
386 #some_user_profile = api_logic.compute_user_profile(some_user_ID)
387 #print(some_user_profile)