· 9 years ago · Jan 20, 2017, 12:34 AM
1#!/usr/bin/env python3
2import pickle
3import os
4import shutil
5import sys
6from os.path import exists
7import hashlib
8
9def ignore_fucking_character_encoding_issues(string):
10 return string.encode(sys.stdout.encoding, errors='replace')
11
12def prompt_create(directory):
13 answer = input("Would you like to create the directory \""+directory+"\"?\n(yes/no/y/n): ").lower()
14 if answer == "y" or answer == 'yes':
15 os.mkdir(directory)
16 elif answer == 'n' or answer == 'no':
17 ...
18 else:
19 print(answer, 'is not a valid answer.')
20 prompt_create(directory)
21
22def paths_exist(source, destination):
23 if os.path.isdir(source) and os.path.isdir(destination):
24 return True
25 else:
26 for path in source, destination:
27 if not os.path.isdir(path):
28 print('"'+path+'"', 'is not a directory.')
29 prompt_create(path)
30 if os.path.isdir(source) and os.path.isdir(destination):
31 return True
32 else:
33 return False
34
35def prepare(path):
36 if not path.endswith('/'):
37 path = path + '/'
38 return path
39
40def clean(subdirectory):
41 if subdirectory == '.':
42 return ''
43 else:
44 return subdirectory[2:]
45
46def build_file_table(path):
47 file_table = {}
48 file_ignore = []
49 dir_ignore = ['__pycache__']
50 file_table['__dirs__'] = []
51 file_table['__abs__'] = path
52 file_ignore = file_ignore + ['.last_sync', 'ignore']
53 for directory in os.walk(path):
54 subdirectory = directory[0]
55 if not subdirectory.endswith('__pycache__'):
56 if not subdirectory == path:
57 if not subdirectory in dir_ignore:
58 file_table['__dirs__'].append(subdirectory[len(path):])
59 for file in directory[2]:
60 if file not in file_ignore:
61 rel_path = subdirectory[len(path):] + '/' + file
62 size, access_time, modify_time = os.stat(path + rel_path)[6:9]
63 file_table[rel_path] = modify_time, size
64 return file_table
65
66def sync_dirs(src, dest):
67 for directory in src['__dirs__']:
68 if directory not in dest['__dirs__']:
69 directory = dest['__abs__'] + directory
70 print("Creating Directory ->", directory)
71 os.mkdir(directory)
72
73def merge_files(src, dest):
74 source = src['__abs__']
75 destination = dest['__abs__']
76 for file in src:
77 if not file.startswith('__'):
78 if file not in dest:
79 print('Duplicating ->', file)
80 shutil.copy(source + file, destination + file)
81
82def hash_file(path):
83 file_hash = hashlib.md5()
84 with open(path, 'rb') as hashme:
85 while True:
86 chunk = hashme.read(1024)
87 if not chunk:
88 break
89 file_hash.update(chunk)
90 return file_hash.hexdigest()
91
92def find_mismatched(src, dest):
93 mismatched = []
94 for file in src:
95 if not file.startswith('__'):
96 if file in dest:
97 print('Hashing =>', file, end=' ')
98 src_digest = hash_file(src['__abs__']+file)
99 print(src_digest, end='')
100 dest_digest = hash_file(dest['__abs__']+file)
101 print('/'+dest_digest)
102 if src_digest != dest_digest:
103 mismatched.append(file)
104 return mismatched
105
106def split_extension(filename):
107 split_name = filename.split('.')
108 if len(split_name) == 1:
109 return split_name[0], ''
110 elif len(split_name) == 2:
111 return split_name[0], split_name[1]
112 else:
113 extension = split_name.pop(-1)
114 filename = '.'.join(split_name)
115 return filename, extension
116
117def get_unique_name(filename, src_path, dest_path):
118 count = 1
119 name, extension = split_extension(filename)
120 name_not_found = True
121 while name_not_found:
122 filename = name+' v'+str(count)+'.'+extension
123 src_name = src_path+filename
124 dest_name = dest_path+filename
125 if exists(src_name) or exists(dest_name):
126 count += 1
127 else:
128 name_not_found = False
129 return filename
130
131def overwrite_handler(mismatched, src, dest):
132 num_mismatched = len(mismatched)
133 if num_mismatched == 1:
134 print('The file', mismatched[0], 'exists in both directories but is not the same.\n')
135 else:
136 print('The following '+str(num_mismatched)+' files exist in both directories but are not the same:')
137 for file in mismatched:
138 print(file)
139 print()
140 print('What would you like to do?')
141 print('1) Overwrite the older files with the newer (more recently modified) files.')
142 print('2) Do nothing and let me sort out these differences on my own.')
143 print(' (Files of the same name must be identical before linking can occur.)')
144 print('3) Make copies of both files in both directories.')
145 choice = input('Enter the number corresponding with your choice: ')
146 if choice == '1':
147 print()
148 confirm = input('Are you sure you want to do this?\nThis action will destroy older files replacing them with the \
149newer ones based on the file\'s modify date, which indicates which file was edited most recently.\n(yes/no/y/n): ').lower()
150 if confirm == 'y' or confirm == 'yes':
151 for file in mismatched:
152 if src[file][0] > dest[file][0]:
153 shutil.copy(src['__abs__'] + file, dest['__abs__'] + file)
154 print(file, 'overwritten in', dest['__abs__'])
155 else:
156 shutil.copy(dest['__abs__'] + file, src['__abs__'] + file)
157 print(file, 'overwritten in', src['__abs__'])
158 return True
159 else:
160 return False
161 elif choice == '2':
162 return False
163 elif choice == '3':
164 print()
165 for file in mismatched:
166 v1 = get_unique_name(file, src['__abs__'], dest['__abs__'])
167 shutil.copy(src['__abs__']+file, src['__abs__']+v1)
168 shutil.copy(src['__abs__']+file, dest['__abs__']+v1)
169 os.remove(src['__abs__']+file)
170 v2 = get_unique_name(file, src['__abs__'], dest['__abs__'])
171 shutil.copy(dest['__abs__']+file, dest['__abs__']+v2)
172 shutil.copy(dest['__abs__']+file, src['__abs__']+v2)
173 os.remove(dest['__abs__']+file)
174 print('Generated', v1, 'and', v2)
175 return True
176 else:
177 print(choice.capitalize(), 'is not a valid choice.')
178 return True if overwrite_handler(mismatched, src, dest) else False
179
180def new_file_tables(source, destination):
181 new_src_file_table = build_file_table(source)
182 new_dest_file_table = build_file_table(destination)
183 return new_src_file_table, new_dest_file_table
184
185def save(src, dest):
186 src_sync_file = open(src['__abs__'] + '.last_sync', 'wb')
187 pickle.dump(src, src_sync_file)
188 dest_sync_file = open(dest['__abs__'] + '.last_sync', 'wb')
189 pickle.dump(dest, dest_sync_file)
190
191def load_file_table(path):
192 table = pickle.load(open(path, 'rb'))
193 return table
194
195def lync(src, dest):
196 source = src['__abs__']
197 destination = dest['__abs__']
198 sync_dirs(src, dest); sync_dirs(dest, src)
199 merge_files(src, dest); merge_files(dest, src)
200 mismatched = find_mismatched(src, dest)
201 num_mismatched = len(mismatched)
202 if num_mismatched >= 1:
203 if overwrite_handler(mismatched, src, dest):
204 new = new_file_tables(source, destination)
205 save(*new)
206 print("Directories have been linked!")
207 else:
208 print('Directories contain different files of the same name so they cannot be linked.')
209 else:
210 new = new_file_tables(source, destination)
211 save(*new)
212 print("Directories have been linked!")
213
214def compare(old, new):
215 created = []
216 deleted= []
217 changed = []
218 for file in old:
219 if file not in new:
220 deleted.append(file)
221 else:
222 if old[file] != new[file] and file != '__dirs__':
223 changed.append(file)
224 [created.append(file) for file in new if file not in old]
225 return created, deleted, changed
226
227def update_tree(src_old, src_new, dest_old, dest_new):
228 for directory in src_new['__dirs__']:
229 if directory not in src_old['__dirs__']:
230 if directory not in dest_new['__dirs__']:
231 os.mkdir(dest_old['__abs__'] + directory)
232 for directory in dest_new['__dirs__']:
233 if directory not in dest_old['__dirs__']:
234 if directory not in src_new['__dirs__']:
235 os.mkdir(src_old['__abs__'] + directory)
236 for directory in src_old['__dirs__']:
237 if directory not in src_new['__dirs__']:
238 shutil.rmtree(dest_old['__abs__'] + directory)
239 for directory in dest_old['__dirs__']:
240 if directory not in dest_new['__dirs__']:
241 shutil.rmtree(src_old['__abs__'] + directory)
242
243def update_leaves(src_changed, dest_changed, src_table, dest_table):
244 ignore = ['__abs__', '__dirs__']
245 for file in src_changed[0]:
246 if file not in ignore:
247 print('Duplicating', file, 'in', dest_table['__abs__'])
248 shutil.copy(src_table['__abs__'] + file, dest_table['__abs__'] + file)
249 for file in dest_changed[0]:
250 if file not in ignore:
251 print('Duplicating', file, 'in', src_table['__abs__'])
252 shutil.copy(dest_table['__abs__'] + file, src_table['__abs__']+file)
253 for file in src_changed[1]:
254 if file not in ignore:
255 print('Deleting', ignore_fucking_character_encoding_issues(file), 'from', dest_table['__abs__'])
256 try: os.remove(dest_table['__abs__'] + file)
257 except: print(file, 'has already been deleted.')
258 for file in dest_changed[1]:
259 if file not in ignore:
260 print('Deleting', file, 'from', src_table['__abs__'])
261 try: os.remove(src_table['__abs__'] + file)
262 except: print(file, 'has already been deleted.')
263 for file in src_changed[2]:
264 if file not in ignore:
265 print(file, 'synchronized in', dest_table['__abs__'])
266 shutil.copy(src_table['__abs__'] + file, dest_table['__abs__'] + file)
267 for file in dest_changed[2]:
268 if file not in ignore:
269 print(file, 'synchronized in', src_table['__abs__'])
270 shutil.copy(dest_table['__abs__'] + file, src_table['__abs__'] + file)
271
272def sync(source, destination):
273 if paths_exist(source, destination):
274 source = prepare(source)
275 destination = prepare(destination)
276 source_file_table = build_file_table(source)
277 destination_file_table = build_file_table(destination)
278 src_file_table = source + '.last_sync'
279 dest_file_table = destination + '.last_sync'
280 src_file_table_exists = os.path.exists(src_file_table)
281 dest_file_table_exists = os.path.exists(dest_file_table)
282 if not src_file_table_exists or not dest_file_table_exists:
283 if not src_file_table_exists and not dest_file_table_exists:
284 print('Neither source nor destination file tables were found.\nLinking Directories...\n')
285 elif not src_file_table_exists:
286 print('Source file table not found.\nLinking Directories...\n')
287 elif not dest_file_table_exists:
288 print('Destination file table not found.\nLinking Directories...\n')
289 lync(source_file_table, destination_file_table)
290 else:
291 print("Synchronizing Directories...")
292 old_table1 = load_file_table(src_file_table)
293 old_table2 = load_file_table(dest_file_table)
294 update_tree(old_table1, source_file_table, old_table2, destination_file_table)
295 src_changed = compare(old_table1, source_file_table)
296 dest_changed = compare(old_table2, destination_file_table)
297 if len(src_changed[0]) != 0 or len(dest_changed[0]) != 0 or len(src_changed[1]) != 0 or len(dest_changed[1]) != 0 or len(src_changed[2]) != 0 or len(dest_changed[2]) != 0:
298 trouble_makers = [x for x in src_changed[2] if x in dest_changed[2]]
299 num_trouble_makers = len(trouble_makers)
300 for file in trouble_makers:
301 src_changed[2].pop(src_changed[2].index(file))
302 dest_changed[2].pop(dest_changed[2].index(file))
303 update_leaves(src_changed, dest_changed,source_file_table, destination_file_table)
304 if num_trouble_makers >= 1:
305 if overwrite_handler(trouble_makers, source_file_table, destination_file_table):
306 new = new_file_tables(source, destination)
307 save(*new)
308 else:
309 #do something crazy
310 new = new_file_tables(source, destination)
311 new_file_table_src = new[0]
312 new_file_table_dest = new[1]
313 for file in trouble_makers:
314 new_file_table_src[file] = old_table1[file]
315 new_file_table_dest[file] = old_table2[file]
316 save(new_file_table_src, new_file_table_dest)
317 else:
318 new = new_file_tables(source, destination)
319 save(*new)
320 else:
321 print('Directories are synchronized!\nNothing to do.')
322 else:
323 print('Cannot synchronize directories.')
324
325
326if __name__ == '__main__':
327 args = sys.argv
328 if len(args) == 1:
329 prompt = input()
330 elif len(args) == 3:
331 sync(args[1], args[2])
332 else:
333 print('Invalid number of arguments. You passed these arguments into the program:')
334 for arg in sys.argv[1:]:
335 print(arg)