· 4 years ago · May 16, 2021, 08:22 AM
1from PyQt5 import QtWidgets, uic
2from PyQt5.Qt import QObject, QRunnable, QThreadPool
3from PyQt5.QtCore import pyqtSignal, pyqtSlot
4from functools import partial
5from sqlite3 import Error
6import concurrent.futures
7import hashlib
8import os
9import pathlib
10import shutil
11import sqlite3
12import subprocess
13import sys
14import tempfile
15import time
16import traceback
17
18sqliteconnection = sqlite3.connect('/home/plutonergy/Documents/sneaky_compiler_v2.sqlite')
19sqlitecursor = sqliteconnection.cursor()
20
21white_extensions = ['.ui', '.py', '.so', '.pyx', '.zip', '.txt', '.ini']
22white_dirs = ['gui', 'img', 'mkm_expansion_data']
23black_dirs = ['__pycache__']
24
25global techdict
26techdict = {}
27
28class WorkerSignals(QObject):
29 finished = pyqtSignal()
30 error = pyqtSignal(tuple)
31 result = pyqtSignal(object)
32 progress = pyqtSignal(int)
33
34class Worker(QRunnable):
35 def __init__(self, function):
36 super(Worker, self).__init__()
37 self.fn = function
38 self.signals = WorkerSignals()
39
40 @pyqtSlot()
41 def run(self):
42 try:
43 result = self.fn()
44 except:
45 traceback.print_exc()
46 exctype, value = sys.exc_info()[:2]
47 self.signals.error.emit((exctype, value, traceback.format_exc()))
48 else:
49 self.signals.result.emit(result) # Return the result of the processing
50 finally:
51 self.signals.finished.emit() # Done
52
53def sqlite_superfunction(connection, cursor, table, column, type):
54 global techdict
55
56 try: return techdict[connection][table][column]
57 except KeyError:
58 if connection not in techdict:
59 techdict.update({connection: {}})
60 if table not in techdict[connection]:
61 techdict[connection].update({table : {}})
62
63 query_one = 'select * from ' + table
64 try: cursor.execute(query_one)
65 except Error:
66
67 with connection:
68 query_two = 'create table ' + table + ' (id INTEGER PRIMARY KEY AUTOINCREMENT)'
69 cursor.execute(query_two)
70
71 if table == 'settings':
72 query_three = 'insert into settings values(?)'
73 sqlitecursor.execute(query_three, (None,))
74
75 cursor.execute(query_one)
76
77 col_names = cursor.description
78
79 for count, row in enumerate(col_names):
80 if row[0] not in techdict[connection][table]:
81 techdict[connection][table].update({row[0] : count})
82
83 try: return techdict[connection][table][column]
84 except KeyError:
85 with connection:
86 query = 'alter table ' + table + ' add column ' + column + ' ' + type.upper()
87 cursor.execute(query)
88
89 return len(col_names)
90
91def db_sqlite(table, column, type='text'):
92 return sqlite_superfunction(sqliteconnection, sqlitecursor, table, column, type)
93
94class DB: # local database
95
96 location = db_sqlite('local_files', 'location')
97 md5 = db_sqlite('local_files', 'md5')
98
99 copy_pyx_files = db_sqlite('settings', 'copy_pyx_files', 'integer')
100 copy_ini_files = db_sqlite('settings', 'copy_ini_files', 'integer')
101 force_update = db_sqlite('settings', 'force_update', 'integer')
102 program_name = db_sqlite('settings', 'program_name')
103 presets = db_sqlite('settings', 'presets')
104 recent_job = db_sqlite('settings', 'recent_job')
105
106class tech:
107 global techdict
108 @staticmethod
109 def md5_hash(local_path):
110 hash_md5 = hashlib.md5()
111 with open(local_path, "rb") as f:
112 for chunk in iter(lambda: f.read(4096), b""):
113 hash_md5.update(chunk)
114 return hash_md5.hexdigest()
115 @staticmethod
116 def save_setting(setting_column, object):
117 """
118 updates sqlite settings table with the column and bool or other value from object
119 if object is string, that works!
120 :param setting_column: string
121 :param object: any Qt-object or string
122 """
123 def update(column, value):
124 with sqliteconnection:
125 query = f'update settings set {column} = (?) where id is 1'
126 sqlitecursor.execute(query, (value,))
127
128 if type(object) == str:
129 update(setting_column, object)
130
131 try:
132 if object.isChecked():
133 update(setting_column, object.isChecked())
134 else:
135 update(setting_column, object.isChecked())
136 except:
137 pass
138
139 try:
140 if object.currentText():
141 update(setting_column, object.currentText())
142 except:
143 pass
144
145 try:
146 if object.toPlainText():
147 update(setting_column, object.toPlainText())
148 except:
149 pass
150
151 @staticmethod
152 def retrieve_setting(index):
153 """
154 :param index: integer
155 :return: column
156 """
157 sqlitecursor.execute('select * from settings where id is 1')
158 data = sqlitecursor.fetchone()
159 if data:
160 return data[index]
161
162 @staticmethod
163 def empty_insert_query(cursor, table):
164 cursor.execute('PRAGMA table_info("{}")'.format(table,))
165 tables = cursor.fetchall()
166 query_part1 = "insert into " + table + " values"
167 query_part2 = "(" + ','.join(['?']*len(tables)) + ")"
168 values = [None] * len(tables)
169 return query_part1 + query_part2, values
170
171def create_shared_object(path):
172 subprocess.run(['cythonize', '-i', '-3', path])
173
174class main(QtWidgets.QMainWindow):
175 def __init__(self):
176 super(main, self).__init__()
177 uic.loadUi('main_program_v1.ui', self)
178 self.move(1200,150)
179 self.setStyleSheet('background-color: gray; color: black')
180 self.threadpool_main = QThreadPool()
181 self.threadpool_status = QThreadPool()
182 self.status_bar = self.statusBar()
183 self.preset_settings()
184
185 # TRIGGERS >
186 self.btn_kill.clicked.connect(self.delete_preset)
187 self.btn_save_preset.clicked.connect(self.save_current_job)
188 self.btn_start_compile.clicked.connect(self.start_compiling)
189 self.check_copy_pyx.clicked.connect(partial(tech.save_setting, 'copy_pyx_files', self.check_copy_pyx))
190 self.check_copy_ini.clicked.connect(partial(tech.save_setting, 'copy_ini_files', self.check_copy_ini))
191 self.check_force.clicked.connect(partial(tech.save_setting, 'force_update', self.check_force))
192 self.combo_name.currentIndexChanged.connect(self.load_preset)
193 # TRIGGER <
194
195 self.show()
196
197 def mousePressEvent(self, QMouseEvent):
198 self.load_preset()
199
200 def delete_preset(self):
201 """
202 deletes the current preset from presets
203 """
204 name = self.combo_name.currentText()
205 data = tech.retrieve_setting(DB.presets)
206
207 if data:
208 presets = data.split('\n')
209 for c in range(len(presets)-1,-1,-1):
210 if presets[c].find(name) > -1 and presets[c][0:len(name) +1] == f'{name},':
211 presets.pop(c)
212 break
213
214 values = '\n'.join(presets)
215 with sqliteconnection:
216 sqlitecursor.execute('update settings set presets = (?) where id is 1', (values,))
217
218 def preset_settings(self):
219 """
220 set gui according to sqlite settings table
221 :return:
222 """
223 cycle = {
224 self.check_force: 'force_update',
225 self.combo_name: 'program_name',
226 self.check_copy_pyx: 'copy_pyx_files',
227 self.check_copy_ini: 'copy_ini_files'
228 }
229
230 for widget, string in cycle.items():
231 rv = tech.retrieve_setting(getattr(DB, string))
232 if rv:
233 if rv == True or rv == False:
234 widget.setChecked(rv)
235
236 elif type(rv) == str:
237 try:
238 widget.setCurrentText(rv)
239 continue
240 except:
241 pass
242
243 try:
244 widget.setPlainText(rv)
245 continue
246 except:
247 pass
248
249 data = tech.retrieve_setting(DB.presets)
250 if data:
251 presets = data.split('\n')
252 saves = []
253 for i in presets:
254 j = i.split(',')
255 saves.append(j[0])
256 saves.sort()
257 self.combo_name.clear()
258 for ii in saves:
259 self.combo_name.addItem(ii)
260
261 recent_job = tech.retrieve_setting(DB.recent_job)
262 if recent_job:
263 for count, ii in enumerate(saves):
264 if recent_job == ii:
265 self.combo_name.setCurrentIndex(count)
266 self.load_preset()
267
268 def load_preset(self):
269 """
270 looks in sqlitedatabase for the name thats in the combobox and sets the
271 data accordingly, its a CSV 0: name, 1: source, 2: destination
272 """
273 name = self.combo_name.currentText()
274 data = tech.retrieve_setting(DB.presets)
275 if data:
276 presets = data.split('\n')
277 for c in range(len(presets)-1,-1,-1):
278 if presets[c].find(name) > -1 and presets[c][0:len(name) +1] == f'{name},':
279 this_job = presets[c].split(',')
280 self.te_source.setPlainText(this_job[1])
281 self.te_dest.setPlainText(this_job[2])
282 tech.save_setting('recent_job', name)
283 break
284
285 def delete_and_fresh_copy(self, source_path, new_path):
286 """
287 deletes tmporary working directory and copies
288 the entire tree from source directory here
289 :param source_path: string
290 :param new_path: string
291 """
292 if os.path.exists(new_path):
293 shutil.rmtree(new_path)
294 shutil.copytree(source_path, new_path)
295
296 def find_files_of_interest(self, tmp_dir):
297 """
298 returns a list of files that was hit from your white_extensions and white_dirs
299 meaning all trash files are excluded from the list
300 :param tmp_dir: folder as string
301 :return: list of files
302 """
303 save_files = []
304 for walk in os.walk(tmp_dir):
305 for file in walk[2]:
306
307 current_file = f'{walk[0]}/{file}'
308 top_dir = walk[0][walk[0].rfind('/') +1:]
309
310
311 for ext in white_extensions:
312 if file.lower().find(ext) > -1 and file[-len(ext):len(file)].lower() == ext:
313 save_files.append(current_file)
314
315 for folder in white_dirs:
316 if top_dir == folder and current_file not in save_files:
317 save_files.append(current_file)
318
319 return save_files
320
321 def check_if_file_is_interesting(self, file_or_list_of_files):
322 """
323 if file_or_list_of_files == str, returns True if file not exists or md5 is changed
324 if file_or_list_of_files == list, pops files from the list that is SAME as database
325 :param file_or_list_of_files: file as string or list of files
326 :return: bool or list
327 """
328 def quick(file):
329 sqlitecursor.execute('select * from local_files where location = (?)', (file,))
330 data = sqlitecursor.fetchone()
331 if not data:
332 return True
333 else:
334 hash = tech.md5_hash(file)
335 if data[DB.md5] != hash:
336 return True
337
338 if type(file_or_list_of_files) == str:
339
340 if self.check_force.isChecked(): # force recompile
341 return False
342
343 rv = quick(file_or_list_of_files)
344 return rv
345
346 elif type(file_or_list_of_files) == list:
347
348 if self.check_force.isChecked(): # force recompile
349 return file_or_list_of_files
350
351 for c in range(len(file_or_list_of_files) - 1, -1, -1):
352 if not quick(file_or_list_of_files[c]):
353 file_or_list_of_files.pop(c)
354
355 return file_or_list_of_files
356
357 def save_md5_hashes(self, file_list):
358 """
359 save all hashes for the current job
360 :param file_list:
361 """
362 query, values = tech.empty_insert_query(sqlitecursor, 'local_files')
363 for file in file_list:
364 values[DB.location] = file
365 values[DB.md5] = tech.md5_hash(file)
366
367 with sqliteconnection:
368 sqlitecursor.execute('delete from local_files where location = (?)', (file,))
369 sqlitecursor.execute(query, values)
370
371 def determine_which_files_to_be_compiled(self, list_of_files):
372 """
373 :param list_of_files:
374 :return: a list of pyx files
375 """
376 pyx_files = []
377 for i in list_of_files:
378 if len(i) > len('.pyx') and i[-len('.pyx'):len(i)].lower() == '.pyx':
379 pyx_files.append(i)
380 return pyx_files
381
382 def compile_list_of_pyxfiles(self, single_or_list):
383 with concurrent.futures.ProcessPoolExecutor() as executor:
384 for _, _ in zip(single_or_list, executor.map(create_shared_object, single_or_list)):
385 pass
386
387 def delete_unwanted_files_from_tmp_dir(self, all_files, tmp_dir):
388 """
389 removes all unwanted files first, then removes all unwanted dirs
390 :param all_files: list
391 :param tmp_dir: string
392 """
393 for walk in os.walk(tmp_dir):
394 for file in walk[2]:
395 current_file = f'{walk[0]}/{file}'
396 if current_file not in all_files:
397 os.remove(current_file)
398
399 for walk in os.walk(tmp_dir):
400 for dir in walk[1]:
401 iterdir = f'{walk[0]}/{dir}'
402 for walkwalk in os.walk(iterdir):
403 if walkwalk[1] == [] and walkwalk[2] == []:
404 shutil.rmtree(walkwalk[0])
405
406 def remove_ext_files_before_final(self, all_files, ext):
407 """
408 parameter needed, but asks the gui checkboxes if we keep them
409 :param all_files: list
410 :param ext: string
411 :return: list
412 """
413 def remover(all_files, ext):
414 for c in range(len(all_files)-1,-1,-1):
415 if all_files[c].find(ext) > -1 and all_files[c][-len(ext):len(all_files[c])] == ext:
416 all_files.pop(c)
417
418 if not self.check_copy_pyx.isChecked() and ext == '.pyx':
419 remover(all_files, ext)
420
421 if not self.check_copy_ini.isChecked() and ext == '.ini':
422 remover(all_files, ext)
423
424 return all_files
425
426 def copy_tree(self, source_dir, destination_dir):
427 shutil.copytree(source_dir, destination_dir, dirs_exist_ok=True)
428
429 def pre_checking(self, source_path, destination_path):
430 """
431 checks that all paths seems ok
432 :param source_path: string
433 :param destination_path: string
434 :return: bool
435 """
436 if not os.path.exists(source_path):
437 self.status_bar.showMessage('Source path has covid!')
438 return False
439
440 if not os.path.exists(destination_path):
441 if len(destination_path) < 5:
442 self.status_bar.showMessage('Destination path short!')
443 return False
444
445 try:
446 pathlib.Path(destination_path).mkdir(parents=True)
447 if not os.path.exists(destination_path):
448 self.status_bar.showMessage('Destination path shit on floor!')
449 return False
450
451 except:
452 self.status_bar.showMessage('Destination path has rabies!')
453 return False
454
455 if self.combo_name.currentText().replace(" ", "") == "":
456 self.combo_name.setCurrentText('SNEAKY_TMP_FOLDER')
457
458 return True
459
460 def save_current_job(self):
461 """
462 saves in sqlite a string that later is split('\n') and then split again(',')
463 0: name, 1: source, 2: destination
464 """
465 name = self.combo_name.currentText()
466 source_path = self.te_source.toPlainText()
467 destination_path = self.te_dest.toPlainText()
468 presets = []
469
470 data = tech.retrieve_setting(DB.presets)
471
472 if data:
473 presets = data.split('\n')
474 for c in range(len(presets)-1,-1,-1):
475 if presets[c].find(name) > -1 and presets[c][0:len(name) +1] == f'{name},':
476 presets.pop(c)
477 break
478
479 presets.append(f'{name},{source_path},{destination_path}')
480 values = '\n'.join(presets)
481 with sqliteconnection:
482 sqlitecursor.execute('update settings set presets = (?) where id is 1', (values,))
483
484 def start_compiling(self):
485 """
486 everything starts here
487 """
488 source_path = self.te_source.toPlainText()
489 destination_path = self.te_dest.toPlainText()
490
491 if not self.pre_checking(source_path, destination_path):
492 return
493
494 if os.path.exists('/mnt/ramdisk'):
495 tmp_dir = f'/mnt/ramdisk/{self.combo_name.currentText()}'
496 else:
497 tmp_dir = f'{tempfile.gettempdir()}/{self.combo_name.currentText()}'
498
499 def finished(self):
500 complete_files = self.find_files_of_interest(tmp_dir)
501 complete_files = self.remove_ext_files_before_final(complete_files, '.pyx')
502 complete_files = self.remove_ext_files_before_final(complete_files, '.ini')
503 self.delete_unwanted_files_from_tmp_dir(complete_files, tmp_dir)
504 self.copy_tree(tmp_dir, destination_path)
505
506 end_time = time.time() - self.start_time
507 message = f'Completed in: {round(end_time, 2)}s... {len(complete_files)} has been updated!'
508 self.status_bar.showMessage(message)
509 self.save_current_job()
510
511 self.start_time = time.time()
512
513 self.delete_and_fresh_copy(source_path, tmp_dir)
514 save_files = self.find_files_of_interest(tmp_dir)
515 save_files = self.check_if_file_is_interesting(save_files)
516 pyx_files = self.determine_which_files_to_be_compiled(save_files)
517 self.save_md5_hashes(save_files)
518
519 thread = Worker(partial(self.compile_list_of_pyxfiles, pyx_files))
520 thread.signals.finished.connect(partial(finished, self))
521 self.threadpool_main.start(thread)
522
523
524app = QtWidgets.QApplication(sys.argv)
525window = main()
526app.exec_()