Release time distributed model trained on ~250k videos (#226)

* replace td with full set model

* updated model files

* fix patience since it will be used in future

* fix version

* model gets test asset correct!

tests/test_cli.py

@@ -183,7 +183,7 @@ def test_actual_prediction_on_single_video(tmp_path):  # noqa: F811
         pd.read_csv(save_dir / "zamba_predictions.csv", index_col="filepath")
         .idxmax(axis=1)
         .values[0]
-        == "monkey_prosimian"
+        == "blank"
     )
 
 

zamba/models/config.py

@@ -29,7 +29,7 @@
 GPUS_AVAILABLE = torch.cuda.device_count()
 
 WEIGHT_LOOKUP = {
-    "time_distributed": "s3://drivendata-client-zamba/data/results/zamba_classification_retraining/td_small_set_new_frame_selection/version_1/",
+    "time_distributed": "s3://drivendata-client-zamba/data/results/zamba_classification_retraining/td_full_set/version_0/",
     "european": "s3://drivendata-client-zamba/data/results/zamba_v2_classification/european_td_dev_base/version_0/",
     "slowfast": "s3://drivendata-client-zamba/data/results/zamba_v2_classification/experiments/slowfast_small_set_full_size_mdlite/version_2/",
 }

zamba/models/official_models/time_distributed/config.yaml

@@ -32,6 +32,7 @@ video_loader_config:
   megadetector_lite_config:
     confidence: 0.25
     fill_mode: score_sorted
+    frame_batch_size: 24
     image_height: 640
     image_width: 640
     n_frames: 16
@@ -45,4 +46,4 @@ video_loader_config:
   total_frames: 16
 predict_config:
   model_name: time_distributed
-public_checkpoint: time_distributed_f5072dafff.ckpt
+public_checkpoint: time_distributed_845b088aa1.ckpt

zamba/models/official_models/time_distributed/predict_configuration.yaml

@@ -1,20 +1,20 @@
-inference_start_time: '2022-07-11T19:04:08.231520'
+inference_start_time: '2022-09-08T22:11:19.578412'
 model_class: TimeDistributedEfficientNet
 predict_config:
-  batch_size: 1
-  checkpoint: version_1/time_distributed.ckpt
-  data_dir: /home/ubuntu/pjmf-zamba
+  batch_size: 3
+  checkpoint: version_0/time_distributed.ckpt
+  data_dir: ''
   dry_run: false
   gpus: 1
   model_cache_dir: /home/ubuntu/.cache/zamba
-  model_name: null
-  num_workers: 1
+  model_name: time_distributed
+  num_workers: 2
   output_class_names: false
   overwrite: false
   proba_threshold: null
   save: true
-  save_dir: version_1
-  skip_load_validation: true
+  save_dir: version_0
+  skip_load_validation: false
   weight_download_region: us
 species:
 - aardvark
@@ -50,7 +50,7 @@ species:
 - small_cat
 - wild_dog_jackal
 video_loader_config:
-  cache_dir: /tmp/zamba_cache
+  cache_dir: data/zamba-cache
   cleanup_cache: false
   crop_bottom_pixels: 50
   early_bias: false
@@ -65,6 +65,7 @@ video_loader_config:
     confidence: 0.25
     device: cuda
     fill_mode: score_sorted
+    frame_batch_size: 24
     image_height: 640
     image_width: 640
     n_frames: 16

zamba/models/official_models/time_distributed/train_configuration.yaml

@@ -1,4 +1,4 @@
-git_hash: 0967bf1ea1d7a8a3a84f37a991df2ff3fea237e8
+git_hash: 3e8bbdd821af094a2183fa78fdc74f78834f71fb
 model_class: TimeDistributedEfficientNet
 species:
 - aardvark
@@ -43,9 +43,9 @@ train_config:
     train_bn: false
     unfreeze_backbone_at_epoch: 3
     verbose: true
-  batch_size: 1
+  batch_size: 3
   checkpoint: null
-  data_dir: /home/ubuntu/pjmf-zamba
+  data_dir: ''
   dry_run: false
   early_stopping_config:
     mode: max
@@ -57,7 +57,7 @@ train_config:
   max_epochs: null
   model_cache_dir: /home/ubuntu/.cache/zamba
   model_name: time_distributed
-  num_workers: 1
+  num_workers: 2
   overwrite: false
   predict_all_zamba_species: true
   save_dir: /home/ubuntu/pjmf-zamba
@@ -71,9 +71,9 @@ train_config:
   skip_load_validation: true
   split_proportions: null
   weight_download_region: us
-training_start_time: '2022-07-10T20:45:32.618594'
+training_start_time: '2022-08-28T14:35:02.087643'
 video_loader_config:
-  cache_dir: /tmp/zamba_cache
+  cache_dir: data/zamba-cache
   cleanup_cache: false
   crop_bottom_pixels: 50
   early_bias: false
@@ -88,6 +88,7 @@ video_loader_config:
     confidence: 0.25
     device: cuda
     fill_mode: score_sorted
+    frame_batch_size: 24
     image_height: 640
     image_width: 640
     n_frames: 16

zamba/models/official_models/time_distributed/val_metrics.json

@@ -1,136 +1,136 @@
 {
-  "val_loss": 0.07741925865411758,
-  "val_macro_f1": 0.5171980539802129,
-  "val_top_1_accuracy": 0.5964691046658259,
-  "val_top_3_accuracy": 0.776796973518285,
-  "val_top_5_accuracy": 0.862547288776797,
-  "val_top_10_accuracy": 0.9337957124842371,
-  "species/val_accuracy/aardvark": 0.9949558638083228,
-  "species/val_f1/aardvark": 0.5000000000000001,
-  "species/val_precision/aardvark": 0.8,
-  "species/val_recall/aardvark": 0.36363636363636365,
-  "species/val_accuracy/antelope_duiker": 0.9501891551071879,
-  "species/val_f1/antelope_duiker": 0.5485714285714286,
-  "species/val_precision/antelope_duiker": 0.7868852459016393,
-  "species/val_recall/antelope_duiker": 0.42105263157894735,
-  "species/val_accuracy/badger": 0.9836065573770492,
-  "species/val_f1/badger": 0.13333333333333336,
-  "species/val_precision/badger": 0.5,
-  "species/val_recall/badger": 0.07692307692307693,
-  "species/val_accuracy/bat": 0.9968474148802018,
+  "val_loss": 0.03829624876379967,
+  "val_macro_f1": 0.40069426903010186,
+  "val_top_1_accuracy": 0.803049860414853,
+  "val_top_3_accuracy": 0.9375619127667877,
+  "val_top_5_accuracy": 0.9676102422477711,
+  "val_top_10_accuracy": 0.9901539939363011,
+  "species/val_accuracy/aardvark": 0.9997598535106414,
+  "species/val_f1/aardvark": 0.0,
+  "species/val_precision/aardvark": 0.0,
+  "species/val_recall/aardvark": 0.0,
+  "species/val_accuracy/antelope_duiker": 0.9017800858523699,
+  "species/val_f1/antelope_duiker": 0.7739394776841233,
+  "species/val_precision/antelope_duiker": 0.928239973483593,
+  "species/val_recall/antelope_duiker": 0.6636255924170616,
+  "species/val_accuracy/badger": 0.9989793774202264,
+  "species/val_f1/badger": 0.0,
+  "species/val_precision/badger": 0.0,
+  "species/val_recall/badger": 0.0,
+  "species/val_accuracy/bat": 0.9993395971542641,
   "species/val_f1/bat": 0.0,
   "species/val_precision/bat": 0.0,
   "species/val_recall/bat": 0.0,
-  "species/val_accuracy/bird": 0.9577553593947037,
-  "species/val_f1/bird": 0.6298342541436464,
-  "species/val_precision/bird": 0.8382352941176471,
-  "species/val_recall/bird": 0.504424778761062,
-  "species/val_accuracy/blank": 0.9029003783102144,
-  "species/val_f1/blank": 0.5389221556886228,
-  "species/val_precision/blank": 0.7142857142857143,
-  "species/val_recall/blank": 0.4326923076923077,
-  "species/val_accuracy/cattle": 0.9962168978562421,
-  "species/val_f1/cattle": 0.8000000000000002,
-  "species/val_precision/cattle": 0.8,
-  "species/val_recall/cattle": 0.8,
+  "species/val_accuracy/bird": 0.9872422177528293,
+  "species/val_f1/bird": 0.5540398740818467,
+  "species/val_precision/bird": 0.8461538461538461,
+  "species/val_recall/bird": 0.4118564742589704,
+  "species/val_accuracy/blank": 0.8529703118902531,
+  "species/val_f1/blank": 0.8342021528671045,
+  "species/val_precision/blank": 0.7676780262911969,
+  "species/val_recall/blank": 0.9133496405010748,
+  "species/val_accuracy/cattle": 0.9993696154654339,
+  "species/val_f1/cattle": 0.6865671641791045,
+  "species/val_precision/cattle": 0.9583333333333334,
+  "species/val_recall/cattle": 0.5348837209302325,
   "species/val_accuracy/cheetah": 1.0,
   "species/val_f1/cheetah": 0.0,
   "species/val_precision/cheetah": 0.0,
   "species/val_recall/cheetah": 0.0,
-  "species/val_accuracy/chimpanzee_bonobo": 0.9823455233291298,
-  "species/val_f1/chimpanzee_bonobo": 0.8541666666666666,
-  "species/val_precision/chimpanzee_bonobo": 0.9213483146067416,
-  "species/val_recall/chimpanzee_bonobo": 0.7961165048543689,
-  "species/val_accuracy/civet_genet": 0.9728877679697352,
-  "species/val_f1/civet_genet": 0.5057471264367815,
-  "species/val_precision/civet_genet": 0.5945945945945946,
-  "species/val_recall/civet_genet": 0.44,
-  "species/val_accuracy/elephant": 0.9665825977301387,
-  "species/val_f1/elephant": 0.6826347305389221,
-  "species/val_precision/elephant": 0.8260869565217391,
-  "species/val_recall/elephant": 0.5816326530612245,
-  "species/val_accuracy/equid": 0.9949558638083228,
-  "species/val_f1/equid": 0.8918918918918919,
-  "species/val_precision/equid": 0.8918918918918919,
-  "species/val_recall/equid": 0.8918918918918919,
-  "species/val_accuracy/forest_buffalo": 0.9987389659520807,
-  "species/val_f1/forest_buffalo": 0.8571428571428571,
-  "species/val_precision/forest_buffalo": 1.0,
-  "species/val_recall/forest_buffalo": 0.75,
-  "species/val_accuracy/fox": 0.9993694829760403,
+  "species/val_accuracy/chimpanzee_bonobo": 0.9897637558910936,
+  "species/val_f1/chimpanzee_bonobo": 0.8472906403940887,
+  "species/val_precision/chimpanzee_bonobo": 0.9497991967871486,
+  "species/val_recall/chimpanzee_bonobo": 0.7647534357316087,
+  "species/val_accuracy/civet_genet": 0.9918950559841503,
+  "species/val_f1/civet_genet": 0.41810344827586204,
+  "species/val_precision/civet_genet": 0.9603960396039604,
+  "species/val_recall/civet_genet": 0.26721763085399447,
+  "species/val_accuracy/elephant": 0.9961876744814336,
+  "species/val_f1/elephant": 0.9057164068299925,
+  "species/val_precision/elephant": 0.9370199692780338,
+  "species/val_recall/elephant": 0.8764367816091954,
+  "species/val_accuracy/equid": 0.9993095788430942,
+  "species/val_f1/equid": 0.8130081300813008,
+  "species/val_precision/equid": 0.9259259259259259,
+  "species/val_recall/equid": 0.7246376811594203,
+  "species/val_accuracy/forest_buffalo": 0.9992195239095848,
+  "species/val_f1/forest_buffalo": 0.23529411764705882,
+  "species/val_precision/forest_buffalo": 0.6666666666666666,
+  "species/val_recall/forest_buffalo": 0.14285714285714285,
+  "species/val_accuracy/fox": 0.9998198901329811,
   "species/val_f1/fox": 0.0,
   "species/val_precision/fox": 0.0,
   "species/val_recall/fox": 0.0,
-  "species/val_accuracy/giraffe": 0.9993694829760403,
-  "species/val_f1/giraffe": 0.6666666666666666,
-  "species/val_precision/giraffe": 1.0,
-  "species/val_recall/giraffe": 0.5,
-  "species/val_accuracy/gorilla": 0.9955863808322825,
-  "species/val_f1/gorilla": 0.8627450980392156,
-  "species/val_precision/gorilla": 0.8461538461538461,
-  "species/val_recall/gorilla": 0.88,
-  "species/val_accuracy/hare_rabbit": 0.9987389659520807,
-  "species/val_f1/hare_rabbit": 0.8333333333333333,
+  "species/val_accuracy/giraffe": 1.0,
+  "species/val_f1/giraffe": 0.0,
+  "species/val_precision/giraffe": 0.0,
+  "species/val_recall/giraffe": 0.0,
+  "species/val_accuracy/gorilla": 0.9984690661303395,
+  "species/val_f1/gorilla": 0.5405405405405406,
+  "species/val_precision/gorilla": 0.8823529411764706,
+  "species/val_recall/gorilla": 0.38961038961038963,
+  "species/val_accuracy/hare_rabbit": 0.9996998168883019,
+  "species/val_f1/hare_rabbit": 0.2857142857142857,
   "species/val_precision/hare_rabbit": 1.0,
-  "species/val_recall/hare_rabbit": 0.7142857142857143,
-  "species/val_accuracy/hippopotamus": 0.9911727616645649,
-  "species/val_f1/hippopotamus": 0.72,
-  "species/val_precision/hippopotamus": 1.0,
-  "species/val_recall/hippopotamus": 0.5625,
-  "species/val_accuracy/hog": 0.9520807061790668,
-  "species/val_f1/hog": 0.6481481481481481,
-  "species/val_precision/hog": 0.8045977011494253,
-  "species/val_recall/hog": 0.5426356589147286,
-  "species/val_accuracy/human": 0.9817150063051703,
-  "species/val_f1/human": 0.8027210884353743,
-  "species/val_precision/human": 0.9672131147540983,
-  "species/val_recall/human": 0.686046511627907,
-  "species/val_accuracy/hyena": 0.9936948297604036,
-  "species/val_f1/hyena": 0.6428571428571429,
-  "species/val_precision/hyena": 1.0,
-  "species/val_recall/hyena": 0.47368421052631576,
-  "species/val_accuracy/large_flightless_bird": 1.0,
+  "species/val_recall/hare_rabbit": 0.16666666666666666,
+  "species/val_accuracy/hippopotamus": 0.9993095788430942,
+  "species/val_f1/hippopotamus": 0.684931506849315,
+  "species/val_precision/hippopotamus": 0.9615384615384616,
+  "species/val_recall/hippopotamus": 0.5319148936170213,
+  "species/val_accuracy/hog": 0.9867018881517726,
+  "species/val_f1/hog": 0.7520984890878567,
+  "species/val_precision/hog": 0.8028673835125448,
+  "species/val_recall/hog": 0.7073684210526315,
+  "species/val_accuracy/human": 0.9912046348272446,
+  "species/val_f1/human": 0.8837762792542643,
+  "species/val_precision/human": 0.8737254901960785,
+  "species/val_recall/human": 0.8940609951845907,
+  "species/val_accuracy/hyena": 0.9996397802659622,
+  "species/val_f1/hyena": 0.6,
+  "species/val_precision/hyena": 0.9,
+  "species/val_recall/hyena": 0.45,
+  "species/val_accuracy/large_flightless_bird": 0.9999099450664906,
   "species/val_f1/large_flightless_bird": 0.0,
   "species/val_precision/large_flightless_bird": 0.0,
   "species/val_recall/large_flightless_bird": 0.0,
-  "species/val_accuracy/leopard": 0.9974779319041615,
-  "species/val_f1/leopard": 0.8461538461538461,
-  "species/val_precision/leopard": 0.9166666666666666,
-  "species/val_recall/leopard": 0.7857142857142857,
-  "species/val_accuracy/lion": 1.0,
+  "species/val_accuracy/leopard": 0.9993395971542641,
+  "species/val_f1/leopard": 0.8405797101449275,
+  "species/val_precision/leopard": 1.0,
+  "species/val_recall/leopard": 0.725,
+  "species/val_accuracy/lion": 0.9999699816888302,
   "species/val_f1/lion": 0.0,
   "species/val_precision/lion": 0.0,
   "species/val_recall/lion": 0.0,
-  "species/val_accuracy/mongoose": 0.955233291298865,
-  "species/val_f1/mongoose": 0.4892086330935252,
-  "species/val_precision/mongoose": 0.68,
-  "species/val_recall/mongoose": 0.38202247191011235,
-  "species/val_accuracy/monkey_prosimian": 0.9539722572509458,
-  "species/val_f1/monkey_prosimian": 0.5521472392638037,
-  "species/val_precision/monkey_prosimian": 0.8181818181818182,
-  "species/val_recall/monkey_prosimian": 0.4166666666666667,
-  "species/val_accuracy/pangolin": 0.9848675914249685,
-  "species/val_f1/pangolin": 0.7209302325581395,
-  "species/val_precision/pangolin": 0.8857142857142857,
-  "species/val_recall/pangolin": 0.6078431372549019,
-  "species/val_accuracy/porcupine": 0.9615384615384616,
-  "species/val_f1/porcupine": 0.3146067415730337,
-  "species/val_precision/porcupine": 0.4117647058823529,
-  "species/val_recall/porcupine": 0.2545454545454545,
-  "species/val_accuracy/reptile": 0.9754098360655737,
+  "species/val_accuracy/mongoose": 0.9909644883378861,
+  "species/val_f1/mongoose": 0.23017902813299232,
+  "species/val_precision/mongoose": 0.9,
+  "species/val_recall/mongoose": 0.13196480938416422,
+  "species/val_accuracy/monkey_prosimian": 0.9673400774472428,
+  "species/val_f1/monkey_prosimian": 0.8695443645083932,
+  "species/val_precision/monkey_prosimian": 0.9168141592920354,
+  "species/val_recall/monkey_prosimian": 0.8269099201824401,
+  "species/val_accuracy/pangolin": 0.9986491759973584,
+  "species/val_f1/pangolin": 0.08163265306122448,
+  "species/val_precision/pangolin": 0.6666666666666666,
+  "species/val_recall/pangolin": 0.043478260869565216,
+  "species/val_accuracy/porcupine": 0.9945666856782638,
+  "species/val_f1/porcupine": 0.4986149584487534,
+  "species/val_precision/porcupine": 0.9278350515463918,
+  "species/val_recall/porcupine": 0.3409090909090909,
+  "species/val_accuracy/reptile": 0.9992795605319245,
   "species/val_f1/reptile": 0.0,
   "species/val_precision/reptile": 0.0,
   "species/val_recall/reptile": 0.0,
-  "species/val_accuracy/rodent": 0.9344262295081968,
-  "species/val_f1/rodent": 0.13333333333333333,
-  "species/val_precision/rodent": 0.8,
-  "species/val_recall/rodent": 0.07272727272727272,
-  "species/val_accuracy/small_cat": 0.9905422446406053,
-  "species/val_f1/small_cat": 0.5454545454545454,
-  "species/val_precision/small_cat": 0.75,
-  "species/val_recall/small_cat": 0.42857142857142855,
-  "species/val_accuracy/wild_dog_jackal": 0.9899117276166457,
-  "species/val_f1/wild_dog_jackal": 0.8297872340425533,
-  "species/val_precision/wild_dog_jackal": 0.8478260869565217,
-  "species/val_recall/wild_dog_jackal": 0.8125
+  "species/val_accuracy/rodent": 0.9759853510641491,
+  "species/val_f1/rodent": 0.12280701754385966,
+  "species/val_precision/rodent": 0.9180327868852459,
+  "species/val_recall/rodent": 0.06580493537015276,
+  "species/val_accuracy/small_cat": 0.999189505598415,
+  "species/val_f1/small_cat": 0.0,
+  "species/val_precision/small_cat": 0.0,
+  "species/val_recall/small_cat": 0.0,
+  "species/val_accuracy/wild_dog_jackal": 0.9993696154654339,
+  "species/val_f1/wild_dog_jackal": 0.36363636363636365,
+  "species/val_precision/wild_dog_jackal": 0.5,
+  "species/val_recall/wild_dog_jackal": 0.2857142857142857
 }