· 5 years ago · Feb 08, 2021, 01:44 PM
1# Trains a gait recognizer CNN
2# This version uses a custom DataGenerator
3
4__author__ = 'Manuel J Marin-Jimenez'
5__copyright__ = 'April 2020'
6
7import sys, os
8
9os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
10import math
11import numpy as np
12
13import os.path as osp
14from os.path import expanduser
15
16import pathlib
17maindir = pathlib.Path(__file__).parent.absolute()
18if sys.version_info[1] >= 6:
19 sys.path.insert(0, osp.join(maindir,".."))
20else:
21 sys.path.insert(0, str(maindir)+ "/..")
22homedir = expanduser("~")
23
24sys.path.insert(0, homedir + "/research/pygait")
25sys.path.insert(0, homedir + "/research/pygait/mains")
26
27# --------------------------------
28import tensorflow as tf
29theSEED = 232323
30tf.random.set_seed(theSEED)
31config = tf.compat.v1.ConfigProto()
32
33# Don't pre-allocate memory; allocate as-needed
34config.gpu_options.allow_growth = True
35config.gpu_options.per_process_gpu_memory_fraction = 0.90 #gpu_rate # TODO
36nworkers = 0 # TODO
37#tf.compat.v1.disable_eager_execution()
38tf.executing_eagerly()
39graph = tf.Graph()
40graph.as_default()
41
42session = tf.compat.v1.Session(graph=graph, config=config)
43session.as_default()
44# --------------------------------
45
46from tensorboard.plugins.hparams import api as hp
47from tensorflow.keras.utils import plot_model
48from tensorflow.keras import optimizers
49
50#from nets import BasicGaitNet, TrainValTensorBoard
51import deepdish as dd
52from nets import UWYHSemiNet, mj_eerVerifDist, UWYHSemiNet3Mods_Fran
53#from tensorflow.keras.callbacks import ReduceLROnPlateau
54
55from data.dataGeneratorGaitSetTL_3mods import DataGeneratorGait
56from utils.mj_netUtils import mj_findLatestFileModel
57from data.mj_utils import mj_splitTrainValGait
58from data.mj_tumgaid import TUMGAIDdb
59from data.mj_datasetinfo import DatasetInfoTUM
60from callbacks.lr_scheduler import ReduceLROnPlateauOrSlowdown
61from utils.rd_JSONInfo import rd_JSONInfo
62
63from utils.mj_utils import mj_isDebugging
64IS_DEBUG = False
65
66# =============== PROJECTOR FUNCTIONS ===========================
67from nets.mj_utils import mj_register_embedding, mj_save_labels_tsv, mj_save_sprite, mj_save_filters, mj_save_filters3d
68META_DATA_FNAME = 'meta.tsv' # Labels will be stored here
69EMBEDDINGS_TENSOR_NAME = 'embeddings'
70
71# ===============================================================
72
73
74def mj_splitTrainValGaitByInfo(Iof):
75 allPairs = Iof["records"]
76
77 rec2gait = Iof["rec2gait"]
78 rec2vid = Iof["rec2vid"]
79
80 # Split in training/val data, grouped per video
81 rec2lab = {p[0][0]: p[1] for p in allPairs}
82 all_keys = [p[0][0] for p in allPairs] # rec2vid.keys()
83 all_vids = [rec2vid[key] for key in all_keys]
84 uvids = np.unique(all_vids)
85 nvids = len(uvids)
86 np.random.shuffle(uvids)
87 perc = 0.09
88 nval = int(perc * nvids)
89 vids_tr = [uvids[i] for i in range(0, nvids - nval)]
90 vids_val = [uvids[i] for i in range(nvids - nval, nvids)]
91 allRecords_single_tr = []
92 for vix in vids_tr:
93 idx = np.where(all_vids == vix)[0]
94 this_vid_pairs = [allPairs[ix] for ix in idx] # [all_keys[ix] for ix in idx]
95 allRecords_single_tr = allRecords_single_tr + this_vid_pairs
96
97 if len(np.unique([l[1] for l in allRecords_single_tr])) < nclasses:
98 print("More classes needed!")
99 import pdb;
100 pdb.set_trace()
101
102 allRecords_single_val = []
103 for vix in vids_val:
104 idx = np.where(all_vids == vix)[0]
105 this_vid_pairs = [allPairs[ix] for ix in idx] # [all_keys[ix] for ix in idx]
106 allRecords_single_val = allRecords_single_val + this_vid_pairs
107
108 return allRecords_single_tr, allRecords_single_val, rec2gait
109
110
111def mj_computeDistMetrics(model, val_generator, multitask, lab4color):
112 gt_labels = []
113 distances = []
114 all_labs0 = []
115 all_codes0 = []
116 l_sprites = []
117
118 for bix in range(len(val_generator)):
119 # import pdb; pdb.set_trace()
120 tuples, labels = val_generator.__getitem__(bix)
121
122 if multitask:
123 labels = labels[0]
124
125 of0, uof0, gray0, ug0, depth0, ud0 = tuples
126 # labs = np.squeeze(labs)
127
128 codes0 = UWYHSemiNet.encode(model, [of0, gray0, depth0], [uof0, ug0, ud0])
129 if multitask:
130 code0_labs = [int(x) for x in labels] # Transformation not needed in this case
131 else:
132 code0_labs = [lab4color[int(x)] for x in labels] # Use correlative labels for color
133 all_labs0 = all_labs0 + code0_labs
134 if bix == 0:
135 all_codes0 = codes0
136 else:
137 all_codes0 = np.append(all_codes0, codes0, axis=0)
138
139 # Create positive and negative pairs
140 _ulabs = np.unique(code0_labs)
141 difs = []
142 dif_labs = []
143 NnegsPerLab = 3
144 for u in _ulabs:
145 idx_u_pos = np.where(code0_labs == u)[0]
146 np.random.shuffle(idx_u_pos)
147 idx_u_neg = np.where(code0_labs != u)[0]
148 np.random.shuffle(idx_u_neg)
149
150 # One positive for this one
151 if len(idx_u_pos) > 1:
152 i = idx_u_pos[0]
153 j = idx_u_pos[1]
154 if len(difs) > 0:
155 difs = np.vstack((difs, codes0[i] - codes0[j]))
156 else:
157 difs = codes0[i] - codes0[j]
158 dif_labs.append(1)
159
160 # N negatives for this one
161 if len(idx_u_neg) >= NnegsPerLab:
162 i = idx_u_pos[0]
163 for t in range(0, NnegsPerLab):
164 j = idx_u_neg[t]
165 if len(difs) > 0:
166 difs = np.vstack((difs, codes0[i] - codes0[j]))
167 else:
168 difs = codes0[i] - codes0[j]
169 dif_labs.append(0)
170
171 # these_sprites = [gray0[ix, 13] for ix in range(gray0.shape[0])]
172 # l_sprites = l_sprites + these_sprites
173
174 # Compute distance
175 dist = np.linalg.norm(difs, axis=1)
176 if bix == 0:
177 gt_labels = dif_labs
178 distances = dist
179 else:
180 gt_labels = np.append(gt_labels, dif_labs)
181 distances = np.append(distances, dist)
182
183 # Compute EER
184 eer_val, thr_eer_val = mj_eerVerifDist(gt_labels, distances)
185
186 chance = np.sum(gt_labels > 0) / len(gt_labels) # Compute chance
187
188 return distances, eer_val, chance, [all_codes0, all_labs0]
189
190
191def inflateListRecords(allRecords_single_tr):
192 for i in range(len(allRecords_single_tr)):
193 r = allRecords_single_tr[i]
194 r_new = []
195 for j in range(len(r)):
196 if j == 0:
197 t = r[j]
198 r_new.append((t[0], t[1], t[1]))
199 else:
200 r_new.append(r[j])
201
202 allRecords_single_tr[i] = r_new
203
204 return allRecords_single_tr
205
206
207def sign_max(**kwargs):
208 def compute(x):
209 cat_data = tf.stack(x, axis=1)
210 max_pos = tf.math.argmax(tf.math.abs(cat_data), axis=1)
211 max_pos = tf.stack([max_pos, tf.repeat(tf.expand_dims(tf.range(cat_data.shape[-1], dtype=max_pos.dtype), axis=0), tf.shape(max_pos)[0], axis=0)], axis=2)
212 data = tf.gather_nd(cat_data, max_pos, batch_dims=1)
213 return data
214 return tf.keras.layers.Lambda(compute, **kwargs)
215
216
217def sign_max_elu(**kwargs):
218 def compute(x):
219 cat_data = tf.stack(x, axis=1)
220 max_pos = tf.math.argmax(tf.math.abs(cat_data), axis=1)
221 max_pos = tf.stack([max_pos, tf.repeat(tf.expand_dims(tf.range(cat_data.shape[-1], dtype=max_pos.dtype), axis=0), tf.shape(max_pos)[0], axis=0)], axis=2)
222 data = tf.gather_nd(cat_data, max_pos, batch_dims=1)
223 return tf.nn.elu(data)
224 return tf.keras.layers.Lambda(compute, **kwargs)
225
226def lr_scheduler(epoch, lr):
227 if epoch % 20 == 0:
228 return lr * 0.1
229 else:
230 return lr
231
232def trainUWYHGaitNet(datadir="matimdbtum_gaid_N150_of25_60x60_lite", dbbasedir='/home/GAIT_local/SSD', experfix="demo",
233 nclasses=0, lr=0.001, dropout=0.4,
234 experdirbase=".", epochs=5, batchsize=32, optimizer="SGD",
235 ndense_units=2048, margin=0.5, savemodelfreq=2, casenet='B',
236 dynmargin=False, hardnegs=4, loss_weights=[1.0, 1.0],
237 modality="gray", modality0='of',
238 initnet="", use3D=False, softlabel=False,
239 datatype = 1, infodir="", freeze_all=False, nofreeze=False,
240 with_missing=True, mergefun="Maximum", normbfmerge=False, activationfun='relu',
241 logdir="", extra_epochs=0, verbose=0,path_gray=""):
242 """
243 Trains a CNN for gait recognition
244 :param datadir: root dir containing dd files
245 :param experfix: string to customize experiment name
246 :param nclasses: number of classes
247 :param lr: starting learning rate
248 :param dropout: dropout value
249 :param tdim: extra dimension of samples. Usually 50 for OF, and 25 for gray and depth
250 :param epochs: max number of epochs
251 :param batchsize: integer
252 :param optimizer: options like {SGD, Adam,...}
253 :param logdir: path to save Tensorboard info
254 :param ndense_units: number of dense units for last FC layer
255 :param verbose: integer
256 :return: model, experdir, accuracy
257 """
258
259 if use3D:
260 input_shape = [(50, 60, 60), (25, 60, 60, 1), (25, 60, 60, 1)]
261 else:
262 input_shape = [(50, 60, 60), (25, 60, 60), (25, 60, 60)]
263
264 multitask = nclasses > 0
265 number_convolutional_layers = 4
266 if not casenet == 'A':
267 filters_size = [(7, 7), (5, 5), (3, 3), (2, 2)]
268 filters_numbers = [96, 192, 512, 512]
269 else:
270 filters_size = [(7, 7), (5, 5), (3, 3), (2, 2)]
271 filters_numbers = [64, 128, 512, 512]
272
273 weight_decay = 0.00005
274 momentum = 0.9
275
276 optimfun = optimizers.Adam(lr=lr)
277 infix = "_opAdam"
278 if optimizer != "Adam":
279 infix = "_op" + optimizer
280 if optimizer == "SGD":
281 optimfun = optimizers.SGD(lr=lr, momentum=momentum, decay=1e-05)
282 elif optimizer == "AMSGrad":
283 optimfun = optimizers.Adam(lr=lr, amsgrad=True)
284 else:
285 optimfun = eval("optimizers." + optimizer + "(lr=initialLR)")
286
287# reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
288# patience=3, min_lr=0.00001)
289# reduce_lr = ReduceLROnPlateauOrSlowdown(monitor='val_loss', factor=0.2,
290# patience=3, min_lr=0.00001, min_delta=0.1, cooldown=10)
291 reduce_lr = tf.keras.callbacks.LearningRateScheduler(lr_scheduler)
292 es_callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5, verbose=1)
293
294 # if use3D:
295 # infix = "_"+ modality0 + "+" + modality + "3D" + infix
296 # else:
297 # infix = "_"+ modality0 + "+" + modality + infix
298 if not with_missing:
299 infix = "_full"
300
301 if use3D:
302 infix = "_3mods_3D" + infix
303 else:
304 infix = "_3mods" + infix
305
306 if datatype == 2:
307 infix = infix + "_dt2"
308
309 if softlabel:
310 infix = infix + "_sl"
311 smoothlabval = 0.1
312 else:
313 smoothlabval = 0
314
315 if casenet != 'B':
316 infix = infix + "_v"+casenet
317
318 if nofreeze:
319 freeze_convs = False
320 else:
321 freeze_convs = True
322 if initnet != "" and freeze_all:
323 infix = infix + "_frall"
324
325 if nclasses > 0:
326 infix = infix + "_mTask"
327 if loss_weights[0] != 1.0:
328 infix = infix + "V{:.2f}".format(loss_weights[0])
329 if loss_weights[1] != 1.0:
330 infix = infix + "I{:.2f}".format(loss_weights[1])
331
332 if ndense_units != 2048:
333 infix = infix + "_nd{:04d}".format(ndense_units)
334
335 if margin != 0.5:
336 infix = infix + "_mg{:03d}".format(int(margin*100))
337 if dynmargin:
338 infix = infix + "_dyn"
339 if hardnegs != 0:
340 infix = infix + "_hn{:02d}".format(hardnegs)
341
342 if normbfmerge:
343 infix = infix + "_nbm"
344
345 if mergefun != "Maximum":
346 infix = infix + "_mg" + mergefun[0:3]
347
348 infix = infix + "_loadGray_modelFran"
349
350 # Create a TensorBoard instance with the path to the logs directory
351 subdir = experfix + '_datagen{}_bs{:03d}_lr{:0.6f}_dr{:0.2f}'.format(infix, batchsize, lr,
352 dropout) # To be customized
353
354 experdir = osp.join(experdirbase, subdir)
355 if verbose > 0:
356 print(experdir)
357 if not osp.exists(experdir):
358 import os
359 os.makedirs(experdir)
360
361 # Add extra dense layer with dropout?
362 if casenet == 'C':
363 ndense_units = [ndense_units, 256]
364
365 # Custom merge function?
366 from tensorflow.keras.layers import Maximum, Average
367 if mergefun == "Maximum":
368 fMerge = Maximum
369 elif mergefun == "Average":
370 fMerge = Average
371 else:
372 fMerge = eval(mergefun)
373
374 # Prepare model
375 pattern_file = "model-state-{:04d}.hdf5"
376 previous_model = mj_findLatestFileModel(experdir, pattern_file, epoch_max=epochs)
377 if previous_model == "":
378 pattern_file = "model-state-{:04d}_weights.hdf5"
379 previous_model = mj_findLatestFileModel(experdir, pattern_file, epoch_max=epochs)
380 if previous_model != "":
381 dname = osp.dirname(previous_model)
382 bname = osp.basename(previous_model)
383 fpix = bname.find("_weights")
384 mname = bname[0:fpix]
385 previous_model = osp.join(dname, mname+".hdf5")
386
387 print(previous_model)
388 print(type(margin))
389 #import pdb; pdb.set_trace()
390
391 initepoch = 0
392 if previous_model != "":
393 pms = previous_model.split("-")
394 initepoch = int(pms[len(pms) - 1].split(".")[0])
395 print("* Info: a previous model was found. Warming up from it...[{:d}]".format(initepoch))
396 from tensorflow.keras.models import load_model
397 #model = load_model(previous_model, custom_objects={"TripletSemiHardLoss": tfa.losses.TripletSemiHardLoss()})
398 model = UWYHSemiNet3Mods_Fran.loadnet(previous_model)
399
400 #import pdb; pdb.set_trace()
401 else:
402 if initnet != "":
403 print("* Model will be init from: "+initnet)
404
405 model = UWYHSemiNet3Mods_Fran.build_or_load(input_shape, number_convolutional_layers,
406 filters_size, filters_numbers, ndense_units, weight_decay, dropout,
407 optimizer=optimfun, margin=margin, nclasses=nclasses, loss_weights=loss_weights,
408 initnet=initnet, freeze_convs=freeze_convs, use3D=use3D, smoothlabels=smoothlabval,
409 freeze_all=freeze_all, fMerge=fMerge, normbfmerge=normbfmerge, fActivation=activationfun)
410
411 model.summary()
412
413# plot_model(model, to_file=osp.join(experdir, 'model.png'))
414 #model.save(osp.join(experdir, "model-init-{:04d}.hdf5".format(00)))
415
416 # Tensorboard
417 if logdir == "":
418 logdir = experdir
419 # Save checkpoint
420 #chkptname = osp.join(logdir, "weights.{epoch:02d}-{val_loss:.2f}.hdf5")
421 chkptname = osp.join(logdir, "model-weights-{epoch:02d}x{val_loss:.2f}_weights.hdf5")
422 checkpoint = tf.keras.callbacks.ModelCheckpoint(chkptname, save_best_only=True, save_weights_only=True)
423
424 from tensorflow.keras.callbacks import TensorBoard
425
426 tensorboard = TensorBoard(log_dir=logdir, histogram_freq=2, write_graph=True, write_images=False,
427 profile_batch = 5)
428 callbacks = [reduce_lr, tensorboard, checkpoint, es_callback]
429 else: # This case is for parameter tuning
430 # Save checkpoint
431 checkpoint = tf.keras.callbacks.ModelCheckpoint(logdir, save_best_only=True)
432
433 from tensorflow.keras.callbacks import TensorBoard
434 tensorboard = TensorBoard(log_dir=logdir, histogram_freq=0, write_graph=True, write_images=False,
435 profile_batch = 3)
436
437 # hpcallback = hp.KerasCallback("/tmp/mjmarin/logs/hparam_tuning", hparams)
438 hpcall = hp.KerasCallback(logdir, hparams)
439 callbacks = [reduce_lr, tensorboard, checkpoint, hpcall]
440
441
442 if dynmargin or hardnegs > 0:
443 savemodelfreq = 2 # FIXME
444
445
446 # MJ: augment training data
447 exp_lev_tr = 4
448 if not with_missing:
449 exp_lev_tr = 0
450 dataset_info = dd.io.load('/home/GAIT_local/SSD/TUM_GAID_tf/tfimdb_tum_gaid_N150_train_of25_60x60.h5')
451
452 # Find label mapping for training
453 if nclasses > 0:
454 ulabels = np.unique(dataset_info['label'])
455 # Create mapping for labels
456 labmap = {}
457 for ix, lab in enumerate(ulabels):
458 labmap[int(lab)] = ix
459 else:
460 labmap = None
461
462 train_generator = DataGeneratorGait(dataset_info,
463 batch_size=batchsize, labmap=labmap,nmods=3,camera=None,
464 expand_level=exp_lev_tr, use3D=use3D,mode="train",lstm=False,multitask=True)
465 train_dataset = tf.keras.utils.OrderedEnqueuer(train_generator, use_multiprocessing=True)
466 train_dataset.start(workers=8, max_queue_size=20)
467
468 exp_lev_val = 2
469 if not with_missing:
470 exp_lev_val = 0
471 val_generator = DataGeneratorGait(dataset_info, batch_size=batchsize,
472 labmap=labmap, nmods=3,camera=None,
473 expand_level=exp_lev_val, use3D=use3D, mode="val",lstm=False,multitask=True)
474 val_dataset = tf.keras.utils.OrderedEnqueuer(val_generator, use_multiprocessing=True)
475 val_dataset.start(workers=8, max_queue_size=20)
476
477 # Save useful info for recovering the model with different Python versions
478 modelpars = {'filters_size': filters_size,
479 'filters_numbers': filters_numbers,
480 'input_shape': input_shape,
481 'ndense_units': ndense_units,
482 'weight_decay': weight_decay,
483 'dropout': dropout,
484 'optimizer': optimizer,
485 'margin': np.float32(margin),
486 'custom': 'TripletSemiHardLoss',
487 'nclasses': nclasses,
488 'softlabel': smoothlabval,
489 'use3D': use3D,
490 'loss_weights': loss_weights,
491 'fMerge': mergefun,
492 'fActivation': activationfun,
493 'normbfmerge': normbfmerge}
494 dd.io.save(osp.join(experdir, "model-config.hdf5"), modelpars)
495
496 rd_JSONInfo(experdir, input_shape, nclasses, number_convolutional_layers, casenet, weight_decay, momentum, optimizer,
497 use3D, datatype, softlabel, dropout, nofreeze, loss_weights, ndense_units,
498 margin, dynmargin, hardnegs, mergefun, batchsize, lr, filters_size, filters_numbers)
499
500 # Validation labels mapping for projector
501
502 if multitask:
503 lab4color = labmap
504
505 from nets.single_model import SingleGaitModel
506 experdir_gray, filename_gray = osp.split(path_gray)
507 model_gray = SingleGaitModel(experdir_gray)
508 model_gray.load(path_gray)
509 model.layers[5].set_weights(model_gray.model.layers[1].get_weights())
510 model.layers[5].trainable = False
511 for layer in model.layers[5].layers[:]:
512 layer.trainable = False
513
514 print("debug")
515
516 # ---------------------------------------
517 # Train model
518 # --------------------------------------
519 print("* Starting training...")
520 last_lr = lr
521
522 ep_steps = int(epochs / savemodelfreq)
523 if ep_steps > 1: # Save partial models in case the process dies
524 ecum = initepoch
525
526 for eix in range(ep_steps):
527 if ecum >= epochs:
528 print("End of main training.")
529 break
530
531 epochs_ = savemodelfreq
532 if verbose > 1:
533 print(experdir)
534
535 #model, hist = UWYHSemiNet.fit_generator(model, ecum+epochs_, callbacks, train_generator, val_generator,
536 # ecum, None, None, nworkers=nworkers) #ecum, steps_per_epoch, validation_steps)
537 model, hist = UWYHSemiNet.fit_generator(model, ecum+epochs_, callbacks, train_dataset.get(), val_dataset.get(),
538 ecum, steps_per_epoch=train_generator.__len__(), validation_steps=val_generator.__len__(), nworkers=nworkers) #ecum, steps_per_epoch, validation_steps)
539 #ecum += epochs_
540 ecum += len(hist.epoch)
541 if ecum % 5 == 0:
542 model.save(osp.join(experdir, "model-state-{:04d}.hdf5".format(ecum)))
543
544 # Save in such a way that can be recovered from different Python versions
545 model.save_weights(osp.join(experdir, "model-state-{:04d}_weights.hdf5".format(ecum)))
546 if ecum % 25 == 0:
547 checkpoint = tf.train.Checkpoint(model=model)
548 checkpoint.save(file_prefix=osp.join(experdir, "ckpt"))
549
550 last_lr = hist.history["lr"][-1]
551
552 # Export some filters to tensorboard
553 if modality0 == "of":
554 mj_save_filters(model, osp.join(experdir, "metrics"), step=ecum + epochs_, branchname="ofBranch", title="filters-of")
555 elif use3D:
556 mj_save_filters3d(model, osp.join(experdir, "metrics"), step=ecum + epochs_, branchname="ofBranch",
557 title="filters-" + modality0 + "-3d")
558 else:
559 mj_save_filters(model, osp.join(experdir, "metrics"), step=ecum + epochs_, branchname="ofBranch",
560 title="filters-"+modality0)
561
562 if not use3D:
563 mj_save_filters(model, osp.join(experdir, "metrics"), step=ecum + epochs_, branchname="grayBranch",
564 title="filters-"+modality)
565 else:
566 mj_save_filters3d(model, osp.join(experdir, "metrics"), step=ecum + epochs_, branchname="grayBranch",
567 title="filters-"+modality+"-3d")
568
569 if not use3D:
570 mj_save_filters(model, osp.join(experdir, "metrics"), step=ecum + epochs_, branchname="depthBranch",
571 title="filters-depth")
572 else:
573 mj_save_filters3d(model, osp.join(experdir, "metrics"), step=ecum + epochs_, branchname="depthBranch",
574 title="filters-depth-3d")
575
576 # Compute EER metric on VALIDATION data
577 # -------------------------------------
578 if verbose > 0:
579 print("+ Computing metrics...", flush=True)
580
581 distances, eer_val, chance, data_metric = mj_computeDistMetrics(model, val_generator, multitask, lab4color)
582 all_codes0, all_labs0 = data_metric # Unpack
583
584 # Save to tensorboard scalars
585 with tf.summary.create_file_writer(osp.join(experdir, "metrics")).as_default():
586 tf.summary.scalar('eer_val', data=eer_val, step=ecum + epochs_)
587 tf.summary.histogram('dist_hist_val', data=distances, step=ecum + epochs_)
588
589 #chance = np.sum(gt_labels > 0) / len(gt_labels) # Compute chance
590 print("EER-val: {:.6f} [{:.2f}]".format(eer_val, 1.0-chance), flush=True)
591 del distances
592
593 # Save codes to Projector
594 print("Exporting to projector", flush=True)
595 prjdir = osp.join(experdir, "metrics")
596 EMBEDDINGS_FPATH = osp.join(prjdir, EMBEDDINGS_TENSOR_NAME + '.ckpt')
597 STEP = ecum+epochs_
598
599 step_val = 3 # FIXME
600 sprite_path = None
601 imsize = (0, 0)
602 # sprite_path, imsize = mj_save_sprite(l_sprites, "sprite.jpg", prjdir)
603 # del l_sprites
604
605 mj_register_embedding(EMBEDDINGS_TENSOR_NAME, META_DATA_FNAME, prjdir, sprite_path, imsize)
606
607 # Subsample to save space and time
608 all_labs0 = [all_labs0[i] for i in range(0, len(all_labs0), step_val)]
609 all_codes0 = all_codes0[range(0, len(all_codes0), step_val)]
610
611 mj_save_labels_tsv(all_labs0, META_DATA_FNAME, prjdir)
612 tensor_embeddings = tf.Variable(all_codes0, name=EMBEDDINGS_TENSOR_NAME)
613 saver = tf.compat.v1.train.Saver([tensor_embeddings]) # Must pass list or dict
614 saver.save(sess=None, global_step=STEP, save_path=EMBEDDINGS_FPATH)
615 del all_codes0
616
617 tf.keras.backend.clear_session() # Experimental use, see if it solves memory leak
618
619 # TODO stop if accurary is almost perfect
620 tr_accuracy = hist.history["classprob_acc"][-1]
621 if ecum >= epochs/2 and tr_accuracy > 0.990: # DEVELOP!
622 print("* Training accuracy is very high: {}. Stop!".format(tr_accuracy*100))
623 epochs = ecum
624 break
625
626 epochs = ecum # Final number of processed epochs
627 else:
628 #model, hist = UWYHSemiNet.fit_generator(model, epochs, callbacks, train_generator, val_generator,
629 # initepoch, steps_per_epoch, validation_steps)
630 model, hist = UWYHSemiNet.fit_generator(model, epochs, callbacks, train_dataset.get(), val_dataset.get(),
631 initepoch, steps_per_epoch=train_generator.__len__(), validation_steps=val_generator.__len__())
632
633 print("Last epoch: {}".format(epochs))
634 model.save(osp.join(experdir, "model-state-{:04d}.hdf5".format(epochs)))
635 model.save_weights(osp.join(experdir, "model-state-{:04d}_weights.hdf5".format(epochs)))
636
637 # Get val accuracy from history
638 if 'hist' in locals():
639 try:
640 #accuracy = hist.history["val_accuracy"][-1]
641 accuracy = hist.history["val_classprob_acc"][-1]
642 except:
643 accuracy = hist.history["classprob_acc"][-1]
644 else:
645 accuracy = 0.5
646
647 # Fine-tune on remaining validation samples
648 if extra_epochs > 0:
649 if verbose > 0:
650 print("Adding validation samples to training and run for few epochs...")
651 del train_generator
652
653 train_generator = DataGeneratorGait(dataset_info, batch_size=batchsize,
654 labmap=labmap, expand_level=3, use3D=use3D, nmods=3,camera=None,
655 mode="trainval",lstm=False,multitask=True)
656 train_dataset = tf.keras.utils.OrderedEnqueuer(train_generator, use_multiprocessing=True)
657 train_dataset.start(workers=8, max_queue_size=20)
658
659 ft_epochs = epochs + extra_epochs # DEVELOP!
660 if nclasses == 150:
661 new_lr = (10**math.ceil(math.log10(last_lr))) * 0.1 # DEVELOP!
662 else:
663 new_lr = min(10**math.ceil(math.log10(last_lr)), last_lr) # last_lr
664
665 #model, hist = UWYHSemiNet.fit_generator(model, ft_epochs, callbacks, train_generator, val_generator,
666 # epochs, None, None, new_lr=new_lr)
667 model, hist = UWYHSemiNet.fit_generator(model, ft_epochs, callbacks, train_dataset.get(), val_dataset.get(),
668 epochs, steps_per_epoch=train_generator.__len__(), validation_steps=val_generator.__len__(), new_lr=new_lr)
669 # Update num epochs
670 epochs = ft_epochs
671
672 model.save(osp.join(experdir, "model-state-{:04d}.hdf5".format(epochs)))
673
674 return model, experdir, accuracy
675
676
677################# MAIN ################
678if __name__ == "__main__":
679 import argparse
680
681 # Input arguments
682 parser = argparse.ArgumentParser(description='Trains a CNN for gait')
683 parser.add_argument('--tuning', default=False, action='store_true')
684 parser.add_argument('--debug', default=False, action='store_true')
685 parser.add_argument('--use3d', default=False, action='store_true', help="Use 3D convs in 2nd branch?")
686 parser.add_argument('--softlabel', default=False, action='store_true', help="Use soft labels?")
687 parser.add_argument('--freezeall', default=False, action='store_true', help="Freeze all weights?")
688 parser.add_argument('--nofreeze', default=False, action='store_true', help="Avoid freezing any weight?")
689 parser.add_argument('--nomissing', default=False, action='store_true', help="Disable missing modalities?")
690 parser.add_argument('--normbfmerge', default=False, action='store_true', help="Normalize features before merging?")
691 parser.add_argument('--dropout', type=float, required=False,
692 default=0.0,
693 help='Dropout value')
694 parser.add_argument('--lr', type=float, required=False,
695 default=0.001,
696 help='Starting learning rate')
697 parser.add_argument('--datadir', type=str, required=False,
698 default=osp.join(homedir, 'datatum', 'matimdbtum_gaid_N150_of25_60x60_lite/'),
699 help="Full path to data directory")
700 parser.add_argument('--dbbasedir', type=str, required=False,
701 default='/home/GAIT_local/SSD/',
702 help="Relative path to data directory")
703 parser.add_argument('--infodir', type=str, required=False,
704 default="",
705 help="Full path to info dir (h5)")
706 parser.add_argument('--experdir', type=str, required=True,
707 default=osp.join(homedir, 'experiments', 'tumgaid_uwyh'),
708 help="Base path to save results of training")
709 parser.add_argument('--prefix', type=str, required=True,
710 default="demo",
711 help="String to prefix experiment directory name.")
712 parser.add_argument('--bs', type=int, required=False,
713 default=32,
714 help='Batch size')
715 parser.add_argument('--epochs', type=int, required=False,
716 default=5,
717 help='Maximum number of epochs')
718 parser.add_argument('--extraepochs', type=int, required=False,
719 default=10,
720 help='Extra number of epochs to add validation data')
721 parser.add_argument('--nclasses', type=int, required=True,
722 default=0,
723 help='Maximum number of epochs')
724 parser.add_argument('--ndense', type=int, required=False,
725 default=2048,
726 help='Number of dense units')
727 parser.add_argument('--casenet', type=str, required=False,
728 default="B",
729 help="Type of net: A, B")
730 parser.add_argument('--margin', type=float, required=False,
731 default=0.5,
732 help='Margin used in loss')
733 parser.add_argument('--dynmargin', default=False, action='store_true',
734 help="Use dynamic margin?")
735 parser.add_argument('--hn', type=int, required=False,
736 default=0,
737 help='Hard negatives per batch')
738 parser.add_argument('--tdim', type=int, required=False,
739 default=50,
740 help='Number of dimensions in 3rd axis time. E.g. OF=50')
741 parser.add_argument('--optimizer', type=str, required=False,
742 default="SGD",
743 help="Optimizer: SGD, Adam, AMSGrad")
744 parser.add_argument('--mod', type=str, required=False,
745 default="gray",
746 help="Extra modality: gray, depth")
747 parser.add_argument('--mod0', type=str, required=False,
748 default="of",
749 help="Base modality: of, gray, depth")
750 parser.add_argument('--datatype', type=int, required=False,
751 default=2,
752 help='Code 1 (matlab-h5) or 2 (new h5)')
753 parser.add_argument('--wid', type=float, required=False,
754 default=1.0,
755 help='Weight for identification task')
756 parser.add_argument('--wver', type=float, required=False,
757 default=1.0,
758 help='Weight for verification task')
759 parser.add_argument('--initnet', type=str, required=False,
760 default="",
761 help="Path to net to initialize")
762 parser.add_argument('--mergefun', type=str, required=False,
763 default="Maximum",
764 help="Choose: Maximum, Average")
765 parser.add_argument('--activationfun', type=str, required=False,
766 default="relu",
767 help="Choose: Maximum, Average")
768 parser.add_argument("--verbose", type=int,
769 nargs='?', const=False, default=1,
770 help="Whether to enable verbosity of output")
771 parser.add_argument("--path_gray", type=str,
772 nargs='?', const=False, default=1,
773 help="Gray model path")
774 args = parser.parse_args()
775 verbose = args.verbose
776 dropout = args.dropout
777 datadir = args.datadir
778 dbbasedir = args.dbbasedir
779 infodir = args.infodir
780 prefix = args.prefix
781 epochs = args.epochs
782 extraepochs = args.extraepochs
783 batchsize = args.bs
784 nclasses = args.nclasses
785 ndense = args.ndense
786 casenet = args.casenet
787 lr = args.lr
788 tdim = args.tdim
789 margin = args.margin
790 dynmargin = args.dynmargin
791 optimizer = args.optimizer
792 tuning = args.tuning
793 experdirbase = args.experdir
794 modality = args.mod
795 modality0 = args.mod0
796 use3D = args.use3d
797 softlabel= args.softlabel
798 IS_DEBUG = args.debug
799 hardnegs = args.hn
800 freeze_all = args.freezeall
801 nofreeze = args.nofreeze
802 with_missing = not args.nomissing
803 datatype= args.datatype
804 initnet = args.initnet #"/home/mjmarin/experiments/tumgaid_uwyhtri/demo__datagen_of+depth_opSGD_vA_mTaskI0.10_nd0062_mg031_bs032_lr0.001000_dr0.00/model-state-0001.hdf5"
805 mergefun = args.mergefun
806 activationfun = args.activationfun
807 normbfmerge = args.normbfmerge
808 path_gray = args.path_gray
809
810 wid = args.wid
811 wver = args.wver
812
813 if freeze_all and nofreeze:
814 print("Error: you cannot ask for both freezing and not freezing at the same time!")
815 exit(-1)
816
817 # Start the processing
818 if tuning:
819 run_dir_base = osp.join(experdirbase, "logs/"+prefix+"_hparam_tuning/")
820
821 # Experimental part ====================================
822 HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([256, 1024, 2048]))
823 HP_DROPOUT = hp.HParam('dropout', hp.RealInterval(0.3, 0.4))
824 HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['Adam', 'SGD']))
825
826 METRIC_ACCURACY = 'accuracy'
827
828 with tf.summary.create_file_writer(run_dir_base).as_default():
829 hp.hparams_config(
830 hparams=[HP_NUM_UNITS, HP_DROPOUT, HP_OPTIMIZER],
831 metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')],
832 )
833
834 trial = 0
835 for optimizer in HP_OPTIMIZER.domain.values:
836 for num_units in HP_NUM_UNITS.domain.values:
837 for dropout_rate in (HP_DROPOUT.domain.min_value, HP_DROPOUT.domain.max_value):
838 hparams = {
839 HP_NUM_UNITS: num_units,
840 HP_DROPOUT: dropout_rate,
841 HP_OPTIMIZER: optimizer
842 }
843 run_dir = run_dir_base + "run-{:d}".format(trial)
844 with tf.summary.create_file_writer(run_dir).as_default():
845 hp.hparams(hparams) # record the values used in this trial
846 print("* Starting trial {:d}".format(trial))
847 final_model, experdir, val_acc = trainUWYHGaitNet(datadir=datadir, dbbasedir=dbbasedir,
848 experfix=prefix, lr=lr, dropout=dropout_rate,
849 experdirbase=experdirbase,
850 nclasses=nclasses,
851 optimizer=optimizer, logdir=run_dir,
852 epochs=epochs, batchsize=batchsize,
853 margin=margin,
854 ndense_units=num_units,path_gray=path_gray)
855 tf.summary.scalar(METRIC_ACCURACY, val_acc, step=epochs)
856 trial += 1
857 else:
858 final_model, experdir, val_acc = trainUWYHGaitNet(datadir=datadir, dbbasedir=dbbasedir, experfix=prefix, lr=lr,
859 dropout=dropout, experdirbase=experdirbase,
860 nclasses=nclasses, optimizer=optimizer,
861 epochs=epochs, batchsize=batchsize, ndense_units=ndense,
862 margin=margin, casenet=casenet, dynmargin=dynmargin,
863 hardnegs=hardnegs, logdir="", loss_weights=[wver, wid],
864 modality=modality, modality0=modality0,
865 initnet=initnet, use3D=use3D, freeze_all=freeze_all,
866 softlabel=softlabel, datatype=datatype, infodir=infodir,
867 with_missing=with_missing, nofreeze=nofreeze,
868 mergefun=mergefun, normbfmerge=normbfmerge, activationfun=activationfun,
869 extra_epochs=extraepochs, verbose=verbose,path_gray=path_gray)
870
871 final_model.save(osp.join(experdir, "model-final-{:04d}.hdf5".format(epochs)))
872 final_model.save_weights(osp.join(experdir, "model-final-{:04d}_weights.hdf5".format(epochs)))
873
874 print("* End of training: {}".format(experdir))
875