From b08be3d613d1e6a7e7528cf785933cf22b96e80d Mon Sep 17 00:00:00 2001 From: tanviraja24 <128422635+tanviraja24@users.noreply.github.com> Date: Thu, 19 Sep 2024 12:26:47 -0400 Subject: [PATCH] Update S3 Bucket (audio-orcasound-net) + Deploy all hydrophones (#196) * Update from streaming-orcasound-net to audio-orcasound-net bucket * Updated bucket version and re-deployed all 5 hydrophones * Update Readme for InferenceSystem --- InferenceSystem/README.md | 6 +++--- InferenceSystem/demos/hls_reader.py | 8 ++++---- InferenceSystem/deploy-aci.yaml | 10 +++++----- InferenceSystem/deploy/bush-point.yaml | 2 +- InferenceSystem/deploy/orcasound-lab.yaml | 2 +- InferenceSystem/deploy/point-robinson.yaml | 2 +- InferenceSystem/deploy/port-townsend.yaml | 2 +- InferenceSystem/deploy/sunset-bay.yaml | 2 +- InferenceSystem/misc/notesOnHLSConversion.txt | 2 +- InferenceSystem/src/LiveInferenceOrchestrator.py | 3 ++- InferenceSystem/src/globals.py | 10 +++++----- 11 files changed, 25 insertions(+), 24 deletions(-) diff --git a/InferenceSystem/README.md b/InferenceSystem/README.md index 4f5f3dcd..abf70dbe 100644 --- a/InferenceSystem/README.md +++ b/InferenceSystem/README.md @@ -104,7 +104,7 @@ python src/LiveInferenceOrchestrator.py --config ./config/Test/FastAI_LiveHLS_Or You should see the following logs in your terminal. Since this is a Test config, no audio is uploaded to Azure and no metadata is written to CosmosDB. ``` -Listening to location https://s3-us-west-2.amazonaws.com/streaming-orcasound-net/rpi_orcasound_lab +Listening to location https://s3-us-west-2.amazonaws.com/audio-orcasound-net/rpi_orcasound_lab Downloading live879.ts live879.ts: 205kB [00:00, 1.17MB/s] Downloading live880.ts @@ -190,7 +190,7 @@ docker run --rm -it --env-file .env live-inference-system In addition, you should see something similar to the following in your console. ``` -Listening to location https://s3-us-west-2.amazonaws.com/streaming-orcasound-net/rpi_orcasound_lab +Listening to location https://s3-us-west-2.amazonaws.com/audio-orcasound-net/rpi_orcasound_lab Downloading live879.ts live879.ts: 205kB [00:00, 1.17MB/s] Downloading live880.ts @@ -384,5 +384,5 @@ export PODCAST_AZURE_STORAGE_CONNECTION_STRING="" Call the script as follows, substituting appropriate values. ``` -python PrepareDataForPredictionExplorer.py --start_time "2020-07-25 19:15" --end_time "2020-07-25 20:15" --s3_stream https://s3-us-west-2.amazonaws.com/streaming-orcasound-net/rpi_orcasound_lab --model_path --annotation_threshold 0.4 --round_id round5 --dataset_folder +python PrepareDataForPredictionExplorer.py --start_time "2020-07-25 19:15" --end_time "2020-07-25 20:15" --s3_stream https://s3-us-west-2.amazonaws.com/audio-orcasound-net/rpi_orcasound_lab --model_path --annotation_threshold 0.4 --round_id round5 --dataset_folder ``` diff --git a/InferenceSystem/demos/hls_reader.py b/InferenceSystem/demos/hls_reader.py index 47a0fa64..552b4437 100644 --- a/InferenceSystem/demos/hls_reader.py +++ b/InferenceSystem/demos/hls_reader.py @@ -18,10 +18,10 @@ import spectrogram_visualizer ORCASOUND_STREAMS = { - # 'OrcasoundLab': 'https://s3-us-west-2.amazonaws.com/streaming-orcasound-net/rpi_orcasound_lab' - 'BushPoint': 'https://s3-us-west-2.amazonaws.com/streaming-orcasound-net/rpi_bush_point' - # 'PortTownsend': 'https://s3-us-west-2.amazonaws.com/streaming-orcasound-net/rpi_port_townsend' - 'SunsetBay': 'https://s3-us-west-2.amazonaws.com/streaming-orcasound-net/rpi_sunset_bay' + # 'OrcasoundLab': 'https://s3-us-west-2.amazonaws.com/audio-orcasound-net/rpi_orcasound_lab' + 'BushPoint': 'https://s3-us-west-2.amazonaws.com/audio-orcasound-net/rpi_bush_point' + # 'PortTownsend': 'https://s3-us-west-2.amazonaws.com/audio-orcasound-net/rpi_port_townsend' + 'SunsetBay': 'https://s3-us-west-2.amazonaws.com/audio-orcasound-net/rpi_sunset_bay' } dirname = os.getcwd() diff --git a/InferenceSystem/deploy-aci.yaml b/InferenceSystem/deploy-aci.yaml index d8bee941..e8ef294c 100644 --- a/InferenceSystem/deploy-aci.yaml +++ b/InferenceSystem/deploy-aci.yaml @@ -14,7 +14,7 @@ properties: secureValue: '' - name: INFERENCESYSTEM_APPINSIGHTS_CONNECTION_STRING secureValue: '' - image: orcaconservancycr.azurecr.io/live-inference-system:11-15-20.FastAI.R1-12.OrcasoundLab.v0 + image: orcaconservancycr.azurecr.io/live-inference-system:09-18-24.FastAI.R1-12.OrcasoundLab.v0 ports: [] resources: requests: @@ -29,7 +29,7 @@ properties: secureValue: '' - name: INFERENCESYSTEM_APPINSIGHTS_CONNECTION_STRING secureValue: '' - image: orcaconservancycr.azurecr.io/live-inference-system:11-15-20.FastAI.R1-12.PortTownsend.v0 + image: orcaconservancycr.azurecr.io/live-inference-system:09-18-24.FastAI.R1-12.PortTownsend.v0 ports: [] resources: requests: @@ -44,7 +44,7 @@ properties: secureValue: '' - name: INFERENCESYSTEM_APPINSIGHTS_CONNECTION_STRING secureValue: '' - image: orcaconservancycr.azurecr.io/live-inference-system:11-15-20.FastAI.R1-12.BushPoint.v0 + image: orcaconservancycr.azurecr.io/live-inference-system:09-19-24.FastAI.R1-12.BushPoint.v0 ports: [] resources: requests: @@ -59,7 +59,7 @@ properties: secureValue: '' - name: INFERENCESYSTEM_APPINSIGHTS_CONNECTION_STRING secureValue: '' - image: orcaconservancycr.azurecr.io/live-inference-system:11-15-20.FastAI.R1-12.SunsetBay.v0 + image: orcaconservancycr.azurecr.io/live-inference-system:09-19-24.FastAI.R1-12.SunsetBay.v0 ports: [] resources: requests: @@ -74,7 +74,7 @@ properties: secureValue: '' - name: INFERENCESYSTEM_APPINSIGHTS_CONNECTION_STRING secureValue: '' - image: orcaconservancycr.azurecr.io/live-inference-system:11-15-20.FastAI.R1-12.PointRobinson.v0 + image: orcaconservancycr.azurecr.io/live-inference-system:09-18-24.FastAI.R1-12.PointRobinson.v0 ports: [] resources: requests: diff --git a/InferenceSystem/deploy/bush-point.yaml b/InferenceSystem/deploy/bush-point.yaml index 0a6b1ca7..bc71e748 100644 --- a/InferenceSystem/deploy/bush-point.yaml +++ b/InferenceSystem/deploy/bush-point.yaml @@ -15,7 +15,7 @@ spec: spec: containers: - name: inference-system - image: orcaconservancycr.azurecr.io/live-inference-system:11-15-20.FastAI.R1-12.BushPoint.v0 + image: orcaconservancycr.azurecr.io/live-inference-system:09-19-24.FastAI.R1-12.BushPoint.v0 resources: limits: cpu: 1 diff --git a/InferenceSystem/deploy/orcasound-lab.yaml b/InferenceSystem/deploy/orcasound-lab.yaml index c1b02661..36acf360 100644 --- a/InferenceSystem/deploy/orcasound-lab.yaml +++ b/InferenceSystem/deploy/orcasound-lab.yaml @@ -15,7 +15,7 @@ spec: spec: containers: - name: inference-system - image: orcaconservancycr.azurecr.io/live-inference-system:11-15-20.FastAI.R1-12.OrcasoundLab.v0 + image: orcaconservancycr.azurecr.io/live-inference-system:09-18-24.FastAI.R1-12.OrcasoundLab.v0 resources: limits: cpu: 1 diff --git a/InferenceSystem/deploy/point-robinson.yaml b/InferenceSystem/deploy/point-robinson.yaml index 89e7c5eb..5f231e2b 100644 --- a/InferenceSystem/deploy/point-robinson.yaml +++ b/InferenceSystem/deploy/point-robinson.yaml @@ -15,7 +15,7 @@ spec: spec: containers: - name: inference-system - image: orcaconservancycr.azurecr.io/live-inference-system:11-15-20.FastAI.R1-12.PointRobinson.v0 + image: orcaconservancycr.azurecr.io/live-inference-system:09-18-24.FastAI.R1-12.PointRobinson.v0 resources: limits: cpu: 1 diff --git a/InferenceSystem/deploy/port-townsend.yaml b/InferenceSystem/deploy/port-townsend.yaml index 8697b139..e6fd85b4 100644 --- a/InferenceSystem/deploy/port-townsend.yaml +++ b/InferenceSystem/deploy/port-townsend.yaml @@ -15,7 +15,7 @@ spec: spec: containers: - name: inference-system - image: orcaconservancycr.azurecr.io/live-inference-system:11-15-20.FastAI.R1-12.PortTownsend.v0 + image: orcaconservancycr.azurecr.io/live-inference-system:09-18-24.FastAI.R1-12.PortTownsend.v0 resources: limits: cpu: 1 diff --git a/InferenceSystem/deploy/sunset-bay.yaml b/InferenceSystem/deploy/sunset-bay.yaml index 53b8590a..e70a9762 100644 --- a/InferenceSystem/deploy/sunset-bay.yaml +++ b/InferenceSystem/deploy/sunset-bay.yaml @@ -15,7 +15,7 @@ spec: spec: containers: - name: inference-system - image: orcaconservancycr.azurecr.io/live-inference-system:11-15-20.FastAI.R1-12.SunsetBay.v0 + image: orcaconservancycr.azurecr.io/live-inference-system:09-19-24.FastAI.R1-12.SunsetBay.v0 resources: limits: cpu: 1 diff --git a/InferenceSystem/misc/notesOnHLSConversion.txt b/InferenceSystem/misc/notesOnHLSConversion.txt index 5de37674..158fe411 100644 --- a/InferenceSystem/misc/notesOnHLSConversion.txt +++ b/InferenceSystem/misc/notesOnHLSConversion.txt @@ -3,7 +3,7 @@ The relevant bucket tonight was a directory with the UNIX timestamp 1543804333, mkdir 20181202_1920-humpback-biggs_OrcasoundLab cd 20181202_1920-humpback-biggs_OrcasoundLab/ -aws s3 sync s3://streaming-orcasound-net/rpi_orcasound_lab/hls/1543804333 . +aws s3 sync s3://audio-orcasound-net/rpi_orcasound_lab/hls/1543804333 . printf "file '%s'\n" ./*.ts > mylist.txt ffmpeg -f concat -safe 0 -i mylist.txt -c copy all.ts ffmpeg -i all.ts -c:v libx264 -c:a copy -bsf:a aac_adtstoasc output.mp4 diff --git a/InferenceSystem/src/LiveInferenceOrchestrator.py b/InferenceSystem/src/LiveInferenceOrchestrator.py index 1d22f94d..25bbc226 100644 --- a/InferenceSystem/src/LiveInferenceOrchestrator.py +++ b/InferenceSystem/src/LiveInferenceOrchestrator.py @@ -102,6 +102,7 @@ def populate_metadata_json( # logger to app insights app_insights_connection_string = os.getenv('INFERENCESYSTEM_APPINSIGHTS_CONNECTION_STRING') + print("INSTRUMENTATION KEY: ", app_insights_connection_string) logger = logging.getLogger(__name__) if app_insights_connection_string is not None: logger.addHandler(AzureLogHandler(connection_string=app_insights_connection_string)) @@ -142,7 +143,7 @@ def populate_metadata_json( hls_stream_type = config_params["hls_stream_type"] hls_polling_interval = config_params["hls_polling_interval"] hls_hydrophone_id = config_params["hls_hydrophone_id"] - hydrophone_stream_url = 'https://s3-us-west-2.amazonaws.com/streaming-orcasound-net/' + hls_hydrophone_id + hydrophone_stream_url = 'https://s3-us-west-2.amazonaws.com/audio-orcasound-net/' + hls_hydrophone_id if hls_stream_type == "LiveHLS": hls_stream = HLSStream(hydrophone_stream_url, hls_polling_interval, local_dir) diff --git a/InferenceSystem/src/globals.py b/InferenceSystem/src/globals.py index dbd89a2c..94118c0a 100644 --- a/InferenceSystem/src/globals.py +++ b/InferenceSystem/src/globals.py @@ -1,11 +1,11 @@ import datetime S3_STREAM_URLS = { - "orcasound_lab": "https://s3-us-west-2.amazonaws.com/streaming-orcasound-net/rpi_orcasound_lab", - "port_townsend": "https://s3-us-west-2.amazonaws.com/streaming-orcasound-net/rpi_port_townsend", - "bush_point": "https://s3-us-west-2.amazonaws.com/streaming-orcasound-net/rpi_bush_point", - "sunset_bay": "https://s3-us-west-2.amazonaws.com/streaming-orcasound-net/rpi_sunset_bay", - "point_robinson": "https://s3-us-west-2.amazonaws.com/streaming-orcasound-net/rpi_point_robinson", + "orcasound_lab": "https://s3-us-west-2.amazonaws.com/audio-orcasound-net/rpi_orcasound_lab", + "port_townsend": "https://s3-us-west-2.amazonaws.com/audio-orcasound-net/rpi_port_townsend", + "bush_point": "https://s3-us-west-2.amazonaws.com/audio-orcasound-net/rpi_bush_point", + "sunset_bay": "https://s3-us-west-2.amazonaws.com/audio-orcasound-net/rpi_sunset_bay", + "point_robinson": "https://s3-us-west-2.amazonaws.com/audio-orcasound-net/rpi_point_robinson", } # Limits time window (end - start) of negative samples to be downloaded for retraining