tokenizer_naem, not path

claficle/conf/model/base_gewechselt.yaml

@@ -4,5 +4,3 @@ defaults:
 
 source_lang: "en"
 target_lang: ???
-
-tokenizer_path: null # path to save to/load from for trained target lang tokenizer

claficle/conf/setup_data.yaml

@@ -9,7 +9,7 @@ seed: 1
 
 disable_wandb: False
 lang: null
-tokenizer_path: null
+tokenizer_name: null  # name of tokenizer in checkpoints/tokenizers/
 
 # to avoid hydra creating output dirs: https://stackoverflow.com/a/64635492/9889508
 hydra:

claficle/data/oscar.py

@@ -200,14 +200,14 @@ def main(cfg: DictConfig):
     oscar.prepare_data()
 
     # optionally, load the tokenizer and perform tokenization
-    if cfg.tokenizer_path is not None:
+    if cfg.tokenizer_name is not None:
         tokenizer = transformers.AutoTokenizer.from_pretrained(
-            os.path.join("checkpoints", "tokenizers", cfg.tokenizer_path)
+            os.path.join("checkpoints", "tokenizers", cfg.tokenizer_name)
         )
         oscar.set_tokenizer(tokenizer)
         oscar.setup()
     # for english, we can always do the tokenization
-    elif cfg.tokenizer_path is None and cfg.lang == "en":
+    elif cfg.tokenizer_name is None and cfg.lang == "en":
         tokenizer = transformers.AutoTokenizer.from_pretrained("gpt2-large")
         oscar.set_tokenizer(tokenizer)
         oscar.setup()

slurm/data/oscar/setup.array.txt

@@ -1,2 +1,2 @@
-lang=fr tokenizer_path=tokenizer_fr
-lang=de tokenizer_path=tokenizer_de
+lang=fr tokenizer_name=fr_gewechselt
+lang=de tokenizer_name=de_gewechselt