From 2a4e9fc7c662d5e1926373a3a28d5d6108efd886 Mon Sep 17 00:00:00 2001 From: Emily Miller Date: Wed, 14 Sep 2022 16:51:27 -0500 Subject: [PATCH] Release time distributed model trained on ~250k videos (#226) * replace td with full set model * updated model files * fix patience since it will be used in future * fix version * model gets test asset correct! --- tests/test_cli.py | 2 +- zamba/models/config.py | 2 +- .../time_distributed/config.yaml | 3 +- .../predict_configuration.yaml | 19 +- .../time_distributed/train_configuration.yaml | 13 +- .../time_distributed/val_metrics.json | 228 +++++++++--------- 6 files changed, 135 insertions(+), 132 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 452484ac..80aff913 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -183,7 +183,7 @@ def test_actual_prediction_on_single_video(tmp_path): # noqa: F811 pd.read_csv(save_dir / "zamba_predictions.csv", index_col="filepath") .idxmax(axis=1) .values[0] - == "monkey_prosimian" + == "blank" ) diff --git a/zamba/models/config.py b/zamba/models/config.py index 31a95dc8..7a271c63 100644 --- a/zamba/models/config.py +++ b/zamba/models/config.py @@ -29,7 +29,7 @@ GPUS_AVAILABLE = torch.cuda.device_count() WEIGHT_LOOKUP = { - "time_distributed": "s3://drivendata-client-zamba/data/results/zamba_classification_retraining/td_small_set_new_frame_selection/version_1/", + "time_distributed": "s3://drivendata-client-zamba/data/results/zamba_classification_retraining/td_full_set/version_0/", "european": "s3://drivendata-client-zamba/data/results/zamba_v2_classification/european_td_dev_base/version_0/", "slowfast": "s3://drivendata-client-zamba/data/results/zamba_v2_classification/experiments/slowfast_small_set_full_size_mdlite/version_2/", } diff --git a/zamba/models/official_models/time_distributed/config.yaml b/zamba/models/official_models/time_distributed/config.yaml index 084b8492..c00e3fa0 100644 --- a/zamba/models/official_models/time_distributed/config.yaml +++ b/zamba/models/official_models/time_distributed/config.yaml @@ -32,6 +32,7 @@ video_loader_config: megadetector_lite_config: confidence: 0.25 fill_mode: score_sorted + frame_batch_size: 24 image_height: 640 image_width: 640 n_frames: 16 @@ -45,4 +46,4 @@ video_loader_config: total_frames: 16 predict_config: model_name: time_distributed -public_checkpoint: time_distributed_f5072dafff.ckpt +public_checkpoint: time_distributed_845b088aa1.ckpt diff --git a/zamba/models/official_models/time_distributed/predict_configuration.yaml b/zamba/models/official_models/time_distributed/predict_configuration.yaml index aa626ac5..52e017a5 100644 --- a/zamba/models/official_models/time_distributed/predict_configuration.yaml +++ b/zamba/models/official_models/time_distributed/predict_configuration.yaml @@ -1,20 +1,20 @@ -inference_start_time: '2022-07-11T19:04:08.231520' +inference_start_time: '2022-09-08T22:11:19.578412' model_class: TimeDistributedEfficientNet predict_config: - batch_size: 1 - checkpoint: version_1/time_distributed.ckpt - data_dir: /home/ubuntu/pjmf-zamba + batch_size: 3 + checkpoint: version_0/time_distributed.ckpt + data_dir: '' dry_run: false gpus: 1 model_cache_dir: /home/ubuntu/.cache/zamba - model_name: null - num_workers: 1 + model_name: time_distributed + num_workers: 2 output_class_names: false overwrite: false proba_threshold: null save: true - save_dir: version_1 - skip_load_validation: true + save_dir: version_0 + skip_load_validation: false weight_download_region: us species: - aardvark @@ -50,7 +50,7 @@ species: - small_cat - wild_dog_jackal video_loader_config: - cache_dir: /tmp/zamba_cache + cache_dir: data/zamba-cache cleanup_cache: false crop_bottom_pixels: 50 early_bias: false @@ -65,6 +65,7 @@ video_loader_config: confidence: 0.25 device: cuda fill_mode: score_sorted + frame_batch_size: 24 image_height: 640 image_width: 640 n_frames: 16 diff --git a/zamba/models/official_models/time_distributed/train_configuration.yaml b/zamba/models/official_models/time_distributed/train_configuration.yaml index ce28a4d5..d53b5768 100644 --- a/zamba/models/official_models/time_distributed/train_configuration.yaml +++ b/zamba/models/official_models/time_distributed/train_configuration.yaml @@ -1,4 +1,4 @@ -git_hash: 0967bf1ea1d7a8a3a84f37a991df2ff3fea237e8 +git_hash: 3e8bbdd821af094a2183fa78fdc74f78834f71fb model_class: TimeDistributedEfficientNet species: - aardvark @@ -43,9 +43,9 @@ train_config: train_bn: false unfreeze_backbone_at_epoch: 3 verbose: true - batch_size: 1 + batch_size: 3 checkpoint: null - data_dir: /home/ubuntu/pjmf-zamba + data_dir: '' dry_run: false early_stopping_config: mode: max @@ -57,7 +57,7 @@ train_config: max_epochs: null model_cache_dir: /home/ubuntu/.cache/zamba model_name: time_distributed - num_workers: 1 + num_workers: 2 overwrite: false predict_all_zamba_species: true save_dir: /home/ubuntu/pjmf-zamba @@ -71,9 +71,9 @@ train_config: skip_load_validation: true split_proportions: null weight_download_region: us -training_start_time: '2022-07-10T20:45:32.618594' +training_start_time: '2022-08-28T14:35:02.087643' video_loader_config: - cache_dir: /tmp/zamba_cache + cache_dir: data/zamba-cache cleanup_cache: false crop_bottom_pixels: 50 early_bias: false @@ -88,6 +88,7 @@ video_loader_config: confidence: 0.25 device: cuda fill_mode: score_sorted + frame_batch_size: 24 image_height: 640 image_width: 640 n_frames: 16 diff --git a/zamba/models/official_models/time_distributed/val_metrics.json b/zamba/models/official_models/time_distributed/val_metrics.json index 81ee296f..74dc7fb5 100644 --- a/zamba/models/official_models/time_distributed/val_metrics.json +++ b/zamba/models/official_models/time_distributed/val_metrics.json @@ -1,136 +1,136 @@ { - "val_loss": 0.07741925865411758, - "val_macro_f1": 0.5171980539802129, - "val_top_1_accuracy": 0.5964691046658259, - "val_top_3_accuracy": 0.776796973518285, - "val_top_5_accuracy": 0.862547288776797, - "val_top_10_accuracy": 0.9337957124842371, - "species/val_accuracy/aardvark": 0.9949558638083228, - "species/val_f1/aardvark": 0.5000000000000001, - "species/val_precision/aardvark": 0.8, - "species/val_recall/aardvark": 0.36363636363636365, - "species/val_accuracy/antelope_duiker": 0.9501891551071879, - "species/val_f1/antelope_duiker": 0.5485714285714286, - "species/val_precision/antelope_duiker": 0.7868852459016393, - "species/val_recall/antelope_duiker": 0.42105263157894735, - "species/val_accuracy/badger": 0.9836065573770492, - "species/val_f1/badger": 0.13333333333333336, - "species/val_precision/badger": 0.5, - "species/val_recall/badger": 0.07692307692307693, - "species/val_accuracy/bat": 0.9968474148802018, + "val_loss": 0.03829624876379967, + "val_macro_f1": 0.40069426903010186, + "val_top_1_accuracy": 0.803049860414853, + "val_top_3_accuracy": 0.9375619127667877, + "val_top_5_accuracy": 0.9676102422477711, + "val_top_10_accuracy": 0.9901539939363011, + "species/val_accuracy/aardvark": 0.9997598535106414, + "species/val_f1/aardvark": 0.0, + "species/val_precision/aardvark": 0.0, + "species/val_recall/aardvark": 0.0, + "species/val_accuracy/antelope_duiker": 0.9017800858523699, + "species/val_f1/antelope_duiker": 0.7739394776841233, + "species/val_precision/antelope_duiker": 0.928239973483593, + "species/val_recall/antelope_duiker": 0.6636255924170616, + "species/val_accuracy/badger": 0.9989793774202264, + "species/val_f1/badger": 0.0, + "species/val_precision/badger": 0.0, + "species/val_recall/badger": 0.0, + "species/val_accuracy/bat": 0.9993395971542641, "species/val_f1/bat": 0.0, "species/val_precision/bat": 0.0, "species/val_recall/bat": 0.0, - "species/val_accuracy/bird": 0.9577553593947037, - "species/val_f1/bird": 0.6298342541436464, - "species/val_precision/bird": 0.8382352941176471, - "species/val_recall/bird": 0.504424778761062, - "species/val_accuracy/blank": 0.9029003783102144, - "species/val_f1/blank": 0.5389221556886228, - "species/val_precision/blank": 0.7142857142857143, - "species/val_recall/blank": 0.4326923076923077, - "species/val_accuracy/cattle": 0.9962168978562421, - "species/val_f1/cattle": 0.8000000000000002, - "species/val_precision/cattle": 0.8, - "species/val_recall/cattle": 0.8, + "species/val_accuracy/bird": 0.9872422177528293, + "species/val_f1/bird": 0.5540398740818467, + "species/val_precision/bird": 0.8461538461538461, + "species/val_recall/bird": 0.4118564742589704, + "species/val_accuracy/blank": 0.8529703118902531, + "species/val_f1/blank": 0.8342021528671045, + "species/val_precision/blank": 0.7676780262911969, + "species/val_recall/blank": 0.9133496405010748, + "species/val_accuracy/cattle": 0.9993696154654339, + "species/val_f1/cattle": 0.6865671641791045, + "species/val_precision/cattle": 0.9583333333333334, + "species/val_recall/cattle": 0.5348837209302325, "species/val_accuracy/cheetah": 1.0, "species/val_f1/cheetah": 0.0, "species/val_precision/cheetah": 0.0, "species/val_recall/cheetah": 0.0, - "species/val_accuracy/chimpanzee_bonobo": 0.9823455233291298, - "species/val_f1/chimpanzee_bonobo": 0.8541666666666666, - "species/val_precision/chimpanzee_bonobo": 0.9213483146067416, - "species/val_recall/chimpanzee_bonobo": 0.7961165048543689, - "species/val_accuracy/civet_genet": 0.9728877679697352, - "species/val_f1/civet_genet": 0.5057471264367815, - "species/val_precision/civet_genet": 0.5945945945945946, - "species/val_recall/civet_genet": 0.44, - "species/val_accuracy/elephant": 0.9665825977301387, - "species/val_f1/elephant": 0.6826347305389221, - "species/val_precision/elephant": 0.8260869565217391, - "species/val_recall/elephant": 0.5816326530612245, - "species/val_accuracy/equid": 0.9949558638083228, - "species/val_f1/equid": 0.8918918918918919, - "species/val_precision/equid": 0.8918918918918919, - "species/val_recall/equid": 0.8918918918918919, - "species/val_accuracy/forest_buffalo": 0.9987389659520807, - "species/val_f1/forest_buffalo": 0.8571428571428571, - "species/val_precision/forest_buffalo": 1.0, - "species/val_recall/forest_buffalo": 0.75, - "species/val_accuracy/fox": 0.9993694829760403, + "species/val_accuracy/chimpanzee_bonobo": 0.9897637558910936, + "species/val_f1/chimpanzee_bonobo": 0.8472906403940887, + "species/val_precision/chimpanzee_bonobo": 0.9497991967871486, + "species/val_recall/chimpanzee_bonobo": 0.7647534357316087, + "species/val_accuracy/civet_genet": 0.9918950559841503, + "species/val_f1/civet_genet": 0.41810344827586204, + "species/val_precision/civet_genet": 0.9603960396039604, + "species/val_recall/civet_genet": 0.26721763085399447, + "species/val_accuracy/elephant": 0.9961876744814336, + "species/val_f1/elephant": 0.9057164068299925, + "species/val_precision/elephant": 0.9370199692780338, + "species/val_recall/elephant": 0.8764367816091954, + "species/val_accuracy/equid": 0.9993095788430942, + "species/val_f1/equid": 0.8130081300813008, + "species/val_precision/equid": 0.9259259259259259, + "species/val_recall/equid": 0.7246376811594203, + "species/val_accuracy/forest_buffalo": 0.9992195239095848, + "species/val_f1/forest_buffalo": 0.23529411764705882, + "species/val_precision/forest_buffalo": 0.6666666666666666, + "species/val_recall/forest_buffalo": 0.14285714285714285, + "species/val_accuracy/fox": 0.9998198901329811, "species/val_f1/fox": 0.0, "species/val_precision/fox": 0.0, "species/val_recall/fox": 0.0, - "species/val_accuracy/giraffe": 0.9993694829760403, - "species/val_f1/giraffe": 0.6666666666666666, - "species/val_precision/giraffe": 1.0, - "species/val_recall/giraffe": 0.5, - "species/val_accuracy/gorilla": 0.9955863808322825, - "species/val_f1/gorilla": 0.8627450980392156, - "species/val_precision/gorilla": 0.8461538461538461, - "species/val_recall/gorilla": 0.88, - "species/val_accuracy/hare_rabbit": 0.9987389659520807, - "species/val_f1/hare_rabbit": 0.8333333333333333, + "species/val_accuracy/giraffe": 1.0, + "species/val_f1/giraffe": 0.0, + "species/val_precision/giraffe": 0.0, + "species/val_recall/giraffe": 0.0, + "species/val_accuracy/gorilla": 0.9984690661303395, + "species/val_f1/gorilla": 0.5405405405405406, + "species/val_precision/gorilla": 0.8823529411764706, + "species/val_recall/gorilla": 0.38961038961038963, + "species/val_accuracy/hare_rabbit": 0.9996998168883019, + "species/val_f1/hare_rabbit": 0.2857142857142857, "species/val_precision/hare_rabbit": 1.0, - "species/val_recall/hare_rabbit": 0.7142857142857143, - "species/val_accuracy/hippopotamus": 0.9911727616645649, - "species/val_f1/hippopotamus": 0.72, - "species/val_precision/hippopotamus": 1.0, - "species/val_recall/hippopotamus": 0.5625, - "species/val_accuracy/hog": 0.9520807061790668, - "species/val_f1/hog": 0.6481481481481481, - "species/val_precision/hog": 0.8045977011494253, - "species/val_recall/hog": 0.5426356589147286, - "species/val_accuracy/human": 0.9817150063051703, - "species/val_f1/human": 0.8027210884353743, - "species/val_precision/human": 0.9672131147540983, - "species/val_recall/human": 0.686046511627907, - "species/val_accuracy/hyena": 0.9936948297604036, - "species/val_f1/hyena": 0.6428571428571429, - "species/val_precision/hyena": 1.0, - "species/val_recall/hyena": 0.47368421052631576, - "species/val_accuracy/large_flightless_bird": 1.0, + "species/val_recall/hare_rabbit": 0.16666666666666666, + "species/val_accuracy/hippopotamus": 0.9993095788430942, + "species/val_f1/hippopotamus": 0.684931506849315, + "species/val_precision/hippopotamus": 0.9615384615384616, + "species/val_recall/hippopotamus": 0.5319148936170213, + "species/val_accuracy/hog": 0.9867018881517726, + "species/val_f1/hog": 0.7520984890878567, + "species/val_precision/hog": 0.8028673835125448, + "species/val_recall/hog": 0.7073684210526315, + "species/val_accuracy/human": 0.9912046348272446, + "species/val_f1/human": 0.8837762792542643, + "species/val_precision/human": 0.8737254901960785, + "species/val_recall/human": 0.8940609951845907, + "species/val_accuracy/hyena": 0.9996397802659622, + "species/val_f1/hyena": 0.6, + "species/val_precision/hyena": 0.9, + "species/val_recall/hyena": 0.45, + "species/val_accuracy/large_flightless_bird": 0.9999099450664906, "species/val_f1/large_flightless_bird": 0.0, "species/val_precision/large_flightless_bird": 0.0, "species/val_recall/large_flightless_bird": 0.0, - "species/val_accuracy/leopard": 0.9974779319041615, - "species/val_f1/leopard": 0.8461538461538461, - "species/val_precision/leopard": 0.9166666666666666, - "species/val_recall/leopard": 0.7857142857142857, - "species/val_accuracy/lion": 1.0, + "species/val_accuracy/leopard": 0.9993395971542641, + "species/val_f1/leopard": 0.8405797101449275, + "species/val_precision/leopard": 1.0, + "species/val_recall/leopard": 0.725, + "species/val_accuracy/lion": 0.9999699816888302, "species/val_f1/lion": 0.0, "species/val_precision/lion": 0.0, "species/val_recall/lion": 0.0, - "species/val_accuracy/mongoose": 0.955233291298865, - "species/val_f1/mongoose": 0.4892086330935252, - "species/val_precision/mongoose": 0.68, - "species/val_recall/mongoose": 0.38202247191011235, - "species/val_accuracy/monkey_prosimian": 0.9539722572509458, - "species/val_f1/monkey_prosimian": 0.5521472392638037, - "species/val_precision/monkey_prosimian": 0.8181818181818182, - "species/val_recall/monkey_prosimian": 0.4166666666666667, - "species/val_accuracy/pangolin": 0.9848675914249685, - "species/val_f1/pangolin": 0.7209302325581395, - "species/val_precision/pangolin": 0.8857142857142857, - "species/val_recall/pangolin": 0.6078431372549019, - "species/val_accuracy/porcupine": 0.9615384615384616, - "species/val_f1/porcupine": 0.3146067415730337, - "species/val_precision/porcupine": 0.4117647058823529, - "species/val_recall/porcupine": 0.2545454545454545, - "species/val_accuracy/reptile": 0.9754098360655737, + "species/val_accuracy/mongoose": 0.9909644883378861, + "species/val_f1/mongoose": 0.23017902813299232, + "species/val_precision/mongoose": 0.9, + "species/val_recall/mongoose": 0.13196480938416422, + "species/val_accuracy/monkey_prosimian": 0.9673400774472428, + "species/val_f1/monkey_prosimian": 0.8695443645083932, + "species/val_precision/monkey_prosimian": 0.9168141592920354, + "species/val_recall/monkey_prosimian": 0.8269099201824401, + "species/val_accuracy/pangolin": 0.9986491759973584, + "species/val_f1/pangolin": 0.08163265306122448, + "species/val_precision/pangolin": 0.6666666666666666, + "species/val_recall/pangolin": 0.043478260869565216, + "species/val_accuracy/porcupine": 0.9945666856782638, + "species/val_f1/porcupine": 0.4986149584487534, + "species/val_precision/porcupine": 0.9278350515463918, + "species/val_recall/porcupine": 0.3409090909090909, + "species/val_accuracy/reptile": 0.9992795605319245, "species/val_f1/reptile": 0.0, "species/val_precision/reptile": 0.0, "species/val_recall/reptile": 0.0, - "species/val_accuracy/rodent": 0.9344262295081968, - "species/val_f1/rodent": 0.13333333333333333, - "species/val_precision/rodent": 0.8, - "species/val_recall/rodent": 0.07272727272727272, - "species/val_accuracy/small_cat": 0.9905422446406053, - "species/val_f1/small_cat": 0.5454545454545454, - "species/val_precision/small_cat": 0.75, - "species/val_recall/small_cat": 0.42857142857142855, - "species/val_accuracy/wild_dog_jackal": 0.9899117276166457, - "species/val_f1/wild_dog_jackal": 0.8297872340425533, - "species/val_precision/wild_dog_jackal": 0.8478260869565217, - "species/val_recall/wild_dog_jackal": 0.8125 + "species/val_accuracy/rodent": 0.9759853510641491, + "species/val_f1/rodent": 0.12280701754385966, + "species/val_precision/rodent": 0.9180327868852459, + "species/val_recall/rodent": 0.06580493537015276, + "species/val_accuracy/small_cat": 0.999189505598415, + "species/val_f1/small_cat": 0.0, + "species/val_precision/small_cat": 0.0, + "species/val_recall/small_cat": 0.0, + "species/val_accuracy/wild_dog_jackal": 0.9993696154654339, + "species/val_f1/wild_dog_jackal": 0.36363636363636365, + "species/val_precision/wild_dog_jackal": 0.5, + "species/val_recall/wild_dog_jackal": 0.2857142857142857 } \ No newline at end of file