· 7 years ago · Oct 12, 2018, 01:18 PM
1# imports
2import sys
3import socket
4import datetime
5import threading
6import signal
7import json
8import MySQLdb
9import S3
10import os.path
11import os
12import mimetypes
13import Image
14
15# configuration
16LISTEN_HOST = ''
17LISTEN_PORT = 49494
18LISTEN_ADDR = (LISTEN_HOST,LISTEN_PORT)
19BUFSIZE = 4096
20LOG_PATH = 'logs/'
21ACCESS_LOG = 'access_log'
22ERROR_LOG = 'error_log'
23MYSQL_HOST = "localhost"
24MYSQL_PORT = 3306
25MYSQL_USER = "root"
26MYSQL_PASS = ""
27MYSQL_DB = "mydb"
28ITEM_TABLE = "attachment"
29S3_ACCESS_KEY = ''
30S3_SECRET_KEY = ''
31S3_BUCKET = "mybucket"
32S3_NEWFILE_PATH = "uploaded/"
33S3_IMAGE_PATH = "media/images/"
34S3_VIDEO_PATH = "media/images/"
35LOCALTMP_PATH = 'tmp/'
36IMAGE_RESOLUTION = {"small":"320x240","medium":"800x600","large":"1024x768"}
37IMAGE_EXTENSION = ".jpg"
38IMAGE_MIME_TYPE = "image/jpg"
39MAX_FIBO = 4000000 # this var is used for the max length of the fibonacci rule to calculate securekey
40
41# variables
42threads = []
43
44# time handling
45def current_time(string=False):
46 time = datetime.datetime.now()
47
48 if string:
49 time = time.replace(microsecond=0)
50 return str(time)
51 else:
52 return time
53
54# log file handling
55def log_error(msg):
56 time = current_time(True)
57 msg = msg + '\n'
58 f = open(LOG_PATH + ERROR_LOG, 'a')
59 f.write('%s: %s' % (time, msg))
60 f.close()
61
62def log_access(msg):
63 time = current_time(True)
64 msg = msg + '\n'
65 f = open(LOG_PATH + ACCESS_LOG, 'a')
66 f.write('%s: %s' % (time, msg))
67 f.close()
68
69# listen tcp server
70try:
71 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
72except socket.error, msg:
73 log_error(msg(1))
74 sys.exit(1)
75
76
77# init mimetypes
78mimetypes.init()
79
80# cleans up when sigint or when script ends
81def cleanup(*args):
82 print 'Exiting'
83
84 global threads
85 global s
86
87 # Close the server socket
88 s.close()
89 s = None
90
91 # Wait for all threads
92 for t in threads:
93 t.join()
94
95 sys.exit(0)
96
97def getFileExtension(file):
98 ext = os.path.splitext(file)[1]
99 return ext
100
101# handles the socket connections
102class ConnectionHandler(threading.Thread):
103 def __init__(self,(conn,addr)):
104 self.conn = conn
105 self.addr = addr
106 self.s3conn = ''
107 self.secretkey = ''
108 self.articleid = ''
109 self.nrimg = ''
110 self.itemlist = ''
111 self.authenticated = False
112
113 # connect mysql server
114 try:
115 self.db = MySQLdb.connect(MYSQL_HOST, MYSQL_USER, MYSQL_PASS, MYSQL_DB)
116 self.dbcursor = self.db.cursor()
117 except Exception, e:
118 raise e
119
120 threading.Thread.__init__(self)
121
122 def run(self):
123 log_access('Connection from ' + str(self.addr))
124
125 while self.conn:
126 data = self.conn.recv(BUFSIZE)
127 if data:
128 # encode json data received and validate it
129 try:
130 data = json.loads(data)
131 except ValueError, msg:
132 log_error(str(self.addr) + ": " + str(msg))
133 break
134
135 for key,value in data.iteritems():
136 log_access('Received: ' + str(key) + " -> " + str(value))
137
138 if key == "articleid":
139 self.articleid = value
140 elif key == "nrimg":
141 self.nrimg = value
142 elif key == "secretkey":
143 self.secretkey = value
144 else:
145 pass
146
147 # check that articleid and nrimg got data
148 if self.articleid and self.nrimg:
149 newkey = self.checkKey()
150 else:
151 log_error(str(self.addr) + ": could not generate secretkey")
152 break
153
154 if newkey == self.secretkey:
155 log_access(str(self.addr) + " authentication successful")
156 self.authenticated = True
157 else:
158 log_error("Error Authenticating " + str(self.addr) + ". Key was " + str(self.secretkey) + " but needs to be " + str(newkey))
159 break
160
161 if self.authenticated:
162 # query DB for image items
163 # connect to database and receive media info
164 try:
165 self.itemlist = self.queryItems(self.articleid)
166 except Exception, e:
167 raise e
168
169 if self.itemlist:
170 print str(self.itemlist)
171 # connect to S3 and get images
172 self.connectS3()
173 else:
174 log_error("Imagelist is empty!")
175
176 # convert them
177 # update DB
178
179 # close connections
180
181 self.dbcursor.close()
182 self.db.close()
183
184 pass
185
186 else:
187 log_access('Connection closed by ' + str(self.addr))
188 break
189 self.conn.close()
190 self.conn = None
191
192 def queryItems(self, id):
193 itemlist = []
194 rows = ''
195
196 try:
197 self.dbcursor.execute("SELECT * from " + ITEM_TABLE)
198 rows = self.dbcursor.fetchall()
199 except MySQLdb.ProgrammingError, e:
200 log_error(str(e))
201
202 if rows:
203 for row in rows:
204 itemlist.append(row[1])
205 else:
206 log_error("Database result for current query is empty!")
207
208 return itemlist
209
210 def checkKey(self):
211 z = [self.articleid,self.nrimg]
212
213 while z[-1] < MAX_FIBO:
214 z.append(z[-1] + z[-2])
215
216 return sum([x for x in z if x % 2 == 0])
217
218 def connectS3(self):
219 self.s3conn = S3.AWSAuthConnection(S3_ACCESS_KEY, S3_SECRET_KEY)
220 # get bucket list
221 if (self.s3conn.check_bucket_exists(S3_BUCKET).status == 200):
222 print '---- bucket status OK ----'
223 for item in self.itemlist:
224 log_access('--- handling item %s ---' % item)
225 self.handleS3Item(item)
226 else:
227 print '---- bucket doesnt exist ----'
228
229 def handleS3Item(self,item):
230 itempath = S3_NEWFILE_PATH + item
231 localitem = LOCALTMP_PATH + item
232 itemext = getFileExtension(item)
233
234 try:
235 self.getS3Item(itempath,localitem)
236 except Exception, e:
237 log_error(str(e))
238
239 try:
240 filetype = self.handleItemType(itemext)
241 except Exception, e:
242 log_error(str(e))
243
244 if filetype:
245 if filetype == "image":
246 self.handleImage(localitem)
247 elif filetype == "video":
248 self.handleVideo()
249
250 os.unlink(localitem)
251
252 def getS3Item(self,remoteitem,localitem):
253 try:
254 itemdata = self.s3conn.get(S3_BUCKET, remoteitem).object.data
255 except Exception, e:
256 log_error(str(e))
257
258 if itemdata:
259 f = open(localitem, 'wb+')
260 f.write(itemdata)
261 f.close()
262 else:
263 log_error("S3 Object doesnt have data!")
264
265 log_access('--- done getting item %s ---' % localitem)
266
267 def handleItemType(self,itemext):
268 try:
269 mime = mimetypes.types_map[itemext]
270 except Exception, e:
271 log_error(str(e))
272
273 if "image" in mime:
274 return "image"
275 elif "video" in mime:
276 return "video"
277 else:
278 pass
279
280 def handleImage(self, imagefile):
281 log_access('--- opened image %s ---' % imagefile)
282 try:
283 self.convertImage(imagefile)
284 except Exception, e:
285 raise e
286
287
288 def convertImage(self, imagefile):
289
290 imagedata = Image.open(imagefile)
291
292 for k,v in IMAGE_RESOLUTION.iteritems():
293 resizename = k
294 resolution = v.split('x')
295 width = int(resolution[0])
296 height = int(resolution[1])
297
298 try:
299 log_access('--- resizing image %s ---' % imagefile)
300 img = imagedata.resize((width,height), Image.NEAREST)
301 except Exception, e:
302 log_error(e)
303
304 newname = self.generateNewImageName(imagefile, resizename)
305
306 try:
307 log_access('--- trying to save new image %s ---' % newname)
308 img.save(newname)
309 except Exception, e:
310 log_error(e)
311
312 log_access('--- new image created %s ---' % newname)
313
314 try:
315 self.putS3Item(newname,IMAGE_MIME_TYPE)
316 except Exception, e:
317 log_error(e)
318
319 def generateNewImageName(self,imgfile,addname):
320 path, ext = os.path.splitext(imgfile)
321
322 newname = path + "_" + addname + ext
323
324 return newname
325
326 def handleVideo(self):
327 pass
328
329
330 def putS3Item(self, item, contenttype):
331 itemreplace = item.replace(LOCALTMP_PATH,'')
332 s3itempath = S3_IMAGE_PATH + itemreplace
333 itemdata = open(item, 'rb').read()
334
335 try:
336 log_access('--- trying to upload %s ---' % itemreplace)
337 self.s3conn.put(S3_BUCKET,s3itempath, S3.S3Object(itemdata),
338 {'x-amz-acl': 'public-read', 'Content-Type': contenttype})
339 except Exception, e:
340 log_error(e)
341
342 self.delTmpItem(item)
343
344 def delTmpItem(self,item):
345 try:
346 os.unlink(item)
347 except Exception, e:
348 raise e
349
350 log_access("deleted temp file %s " % item)
351
352# runs the actual tcp server
353try:
354 s.bind((LISTEN_HOST, LISTEN_PORT))
355 s.listen(5)
356 log_access('Server started!')
357except socket.error, e:
358 s.close()
359
360if s is None:
361 log_error('could not open socket')
362 sys.exit(1)
363
364# Catch some signals
365signal.signal(signal.SIGINT, cleanup)
366signal.signal(signal.SIGTERM, cleanup)
367
368# builds up the single client connections
369# handle incoming connections
370while s:
371 print "Threads:", len(threads)
372 ch = ConnectionHandler(s.accept())
373 ch.daemon = True
374 ch.start()
375 threads.append(ch)
376
377cleanup()