refactor to better support filename parsing and renaming for clarity.…

… Fixed soundtrap .log.wav bug
mbari-org · Aug 20, 2024 · fe28cf0 · fe28cf0
1 parent da492ea
commit fe28cf0
Show file tree

Hide file tree

Showing 24 changed files with 441 additions and 485 deletions.
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
@@ -45,7 +45,7 @@ plugins: syrupy-4.6.1, cov-4.1.0
 collected 14 items
 
 tests/test_file_helper.py .                                               [  7%]
-tests/test_json_generator.py ss.                                          [ 28%]
+tests/test_meta_generator.py ...                                          [ 28%]
 tests/test_json_support.py ...                                            [ 50%]
 tests/test_metadata.py ...                                                [ 71%]
 tests/test_misc.py ..                                                     [ 85%]

diff --git a/README.md b/README.md
@@ -32,7 +32,7 @@ You can run `python3 --version` to check the version of Python installed.
 
 As a general practice, it is recommended to use a virtual environment for the installation.
 ```shell
-python3.9 -m venv virtenv
+python3.11 -m venv virtenv
 source virtenv/bin/activate
 ```
 
@@ -52,7 +52,7 @@ The package includes the following CLI programs:
 
 | Program | Description                                    |
 | ------- |------------------------------------------------|
-| [`pbp-json-gen`](https://docs.mbari.org/pbp/pbp-json-gen/) | Generate JSON files with audio metadata.       |
+| [`pbp-meta-gen`](https://docs.mbari.org/pbp/pbp-meta-gen/) | Generate JSON files with audio metadata.       |
 | [`pbp`](https://docs.mbari.org/pbp/pbp/) | Main HMB generation program.                   |
 | [`pbp-cloud`](https://docs.mbari.org/pbp/pbp-cloud/) | Program for cloud based processing.            |
 | [`pbp-plot`](https://docs.mbari.org/pbp/pbp-plot/) | Utility program to plot resulting HMB product. |

diff --git a/justfile b/justfile
@@ -28,7 +28,7 @@ to-gizo user="carueda" server="gizo.shore.mbari.org": tgz
 tgz:
     #!/usr/bin/env bash
     HASH=$(git rev-parse --short HEAD)
-    git archive ${HASH} -o pbp_${HASH}.tgz --prefix=pbp/
+    git archive ${HASH} -o pbp_${HASH}.tgz --prefixes=pbp/
 
 # Run main (on gizo)
 main-gizo date="20220902" output_dir="/PAM_Analysis/pypam-space/test_output/daily":
@@ -38,7 +38,7 @@ main-gizo date="20220902" output_dir="/PAM_Analysis/pypam-space/test_output/dail
                  --voltage-multiplier=3 \
                  --sensitivity-uri=misc/icListen1689_sensitivity_hms256kHz.nc \
                  --subset-to 10 100000 \
-                 --audio-path-map-prefix="s3://pacific-sound-256khz-2022~file:///PAM_Archive/2022" \
+                 --audio-path-map-prefixes="s3://pacific-sound-256khz-2022~file:///PAM_Archive/2022" \
                  --output-dir={{output_dir}}
 
 # Run main (on gizo) with some initial test jsons
@@ -50,7 +50,7 @@ main-gizo-test *more_args="":
                  --sensitivity-uri=misc/icListen1689_sensitivity_hms256kHz.nc \
                  --subset-to 10 100000 \
                  --audio-base-dir=tests/wav \
-                 --audio-path-map-prefix="s3://pacific-sound-256khz-2022~file:///PAM_Archive/2022" \
+                 --audio-path-map-prefixes="s3://pacific-sound-256khz-2022~file:///PAM_Archive/2022" \
                  --output-dir=/PAM_Analysis/pypam-space/test_output/daily \
                  {{more_args}}
 
@@ -73,7 +73,7 @@ main-gizo-multiple-days year month *days="":
              --voltage-multiplier=3 \
              --sensitivity-uri=misc/icListen1689_sensitivity_hms256kHz.nc \
              --subset-to 10 100000 \
-             --audio-path-map-prefix="s3://pacific-sound-256khz-{{year}}~file:///PAM_Archive/{{year}}" \
+             --audio-path-map-prefixes="s3://pacific-sound-256khz-{{year}}~file:///PAM_Archive/{{year}}" \
              --output-dir="$output_dir" \
              > "$out" 2>&1 &
     done
@@ -96,7 +96,7 @@ main-mb05 *more_args="":
                  --variable-attrs metadata/mb05/variableAttributes_MB05.yaml \
                  --subset-to 10 24000 \
                  --output-dir=NB_SPACE/OUTPUT \
-                 --output-prefix=MB05_ \
+                 --output-prefixes=MB05_ \
                  --download-dir=NB_SPACE/DOWNLOADS \
                  --assume-downloaded-files \
                  --retain-downloaded-files \
@@ -123,7 +123,7 @@ main-nrs11 date='20200101' *more_args='':
                  --voltage-multiplier=2.5 \
                  --sensitivity-uri="$WS/NRS11_H5R6_sensitivity_hms5kHz.nc" \
                  --subset-to 10 2000 \
-                 --output-prefix=NRS11_ \
+                 --output-prefixes=NRS11_ \
                  --output-dir="$WS/OUTPUT" \
                  --download-dir="$WS/DOWNLOADS" \
                  --retain-downloaded-files \
@@ -149,7 +149,7 @@ main-nrs11-multiple-days year month *days="":
              --voltage-multiplier=2.5 \
              --sensitivity-uri="$WS/NRS11_H5R6_sensitivity_hms5kHz.nc" \
              --subset-to 10 2000 \
-             --output-prefix=NRS11_ \
+             --output-prefixes=NRS11_ \
              --output-dir="$WS/OUTPUT" \
              --download-dir="$WS/DOWNLOADS" \
              --retain-downloaded-files \

diff --git a/pbp/file_helper.py b/pbp/file_helper.py
@@ -207,7 +207,7 @@ def __init__(
           Prefix mapping to get actual audio uri to be used.
           Example: `s3://pacific-sound-256khz-2022~file:///PAM_Archive/2022`
         :param audio_path_prefix:
-          Ad hoc path prefix for sound file locations, e.g. "/Volumes"
+          Ad hoc path prefixes for sound file locations, e.g. "/Volumes"
         :param segment_size_in_mins:
             The size of each audio segment to extract, in minutes. By default, 1.
         :param s3_client:

diff --git a/pbp/json_generator/gen_iclisten.py b/pbp/json_generator/gen_iclisten.py
diff --git a/pbp/json_generator/utils.py b/pbp/json_generator/utils.py
diff --git a/pbp/main_args.py b/pbp/main_args.py
@@ -69,7 +69,7 @@ def parse_arguments():
     )
 
     parser.add_argument(
-        "--audio-path-map-prefix",
+        "--audio-path-map-prefixes",
         type=str,
         metavar="from~to",
         default="",
@@ -78,12 +78,12 @@ def parse_arguments():
     )
 
     parser.add_argument(
-        "--audio-path-prefix",
+        "--audio-path-prefixes",
         type=str,
         metavar="dir",
         default="",
-        help="Ad hoc path prefix for sound file location, for example, /Volumes."
-        " By default, no prefix applied.",
+        help="Ad hoc path prefixes for sound file location, for example, /Volumes."
+        " By default, no prefixes applied.",
     )
 
     parser.add_argument(
@@ -128,11 +128,11 @@ def parse_arguments():
     )
 
     parser.add_argument(
-        "--output-prefix",
+        "--output-prefixes",
         type=str,
-        metavar="prefix",
+        metavar="prefixes",
         default="milli_psd_",
-        help="Output filename prefix",
+        help="Output filename prefixes",
     )
 
     parser.add_argument(