Skip to content

Commit

Permalink
updated raven selection table columns (#306)
Browse files Browse the repository at this point in the history 
* updated raven selection table columns

* Added a list file output for the combined selection tables

---------

Co-authored-by: Josef Haupt <josef.haupt@phil.tu-chemnitz.de>
  • Loading branch information
@Josef-Haupt
Josef-Haupt and Josef Haupt authored Apr 16, 2024
1 parent 750c380 commit 8b1da28
Show file tree
Hide file tree
Showing 2 changed files with 65 additions and 62 deletions.
125 changes: 64 additions & 61 deletions analyze.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Module to analyze audio samples.
"""

import argparse
import datetime
import json
Expand All @@ -17,6 +18,9 @@
import species
import utils

# 0 1 2 3 4 5 6 7 8 9 10 11
RTABLE_HEADER = "Selection\tView\tChannel\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tCommon Name\tSpecies Code\tConfidence\tBegin Path\tFile Offset (s)\n"


def loadCodes():
"""Loads the eBird codes.
Expand Down Expand Up @@ -46,13 +50,11 @@ def saveResultFile(r: dict[str, list], path: str, afile_path: str):
out_string = ""

if cfg.RESULT_TYPE == "table":
# Raven selection header
header = "Selection\tView\tChannel\tBegin Path\tFile Duration (s)\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tSpecies Code\tCommon Name\tConfidence\n"
selection_id = 0
filename = os.path.basename(afile_path)

# Write header
out_string += header
out_string += RTABLE_HEADER

# Read native sample rate
high_freq = audio.get_sample_rate(afile_path) / 2
Expand All @@ -72,38 +74,17 @@ def saveResultFile(r: dict[str, list], path: str, afile_path: str):
if c[1] > cfg.MIN_CONFIDENCE and (not cfg.SPECIES_LIST or c[0] in cfg.SPECIES_LIST):
selection_id += 1
label = cfg.TRANSLATED_LABELS[cfg.LABELS.index(c[0])]
rstring += "{}\tSpectrogram 1\t1\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{:.4f}\n".format(
selection_id,
afile_path,
audio.getAudioFileLength(afile_path, cfg.SAMPLE_RATE),
start,
end,
low_freq,
high_freq,
cfg.CODES[c[0]] if c[0] in cfg.CODES else c[0],
label.split("_", 1)[-1],
c[1],
)
code = cfg.CODES[c[0]] if c[0] in cfg.CODES else c[0]
rstring += f"{selection_id}\tSpectrogram 1\t1\t{start}\t{end}\t{low_freq}\t{high_freq}\t{label.split('_', 1)[-1]}\t{code}\t{c[1]:.4f}\t{afile_path}\t{start}\n"

# Write result string to file
out_string += rstring

# If we don't have any valid predictions, we still need to add a line to the selection table in case we want to combine results
# TODO: That's a weird way to do it, but it works for now. It would be better to keep track of file durations during the analysis.
if len(out_string) == len(header) and cfg.OUTPUT_PATH is not None:
if len(out_string) == len(RTABLE_HEADER) and cfg.OUTPUT_PATH is not None:
selection_id += 1
out_string += "{}\tSpectrogram 1\t1\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{:.4f}\n".format(
selection_id,
afile_path,
audio.getAudioFileLength(afile_path, cfg.SAMPLE_RATE),
0,
3,
low_freq,
high_freq,
"nocall",
"nocall",
1.0,
)
out_string += f"{selection_id}\tSpectrogram 1\t1\t0\t3\t{low_freq}\t{high_freq}\tnocall\tnocall\t1.0\t{afile_path}\t0\n"

elif cfg.RESULT_TYPE == "audacity":
# Audacity timeline labels
Expand All @@ -113,7 +94,9 @@ def saveResultFile(r: dict[str, list], path: str, afile_path: str):
for c in r[timestamp]:
if c[1] > cfg.MIN_CONFIDENCE and (not cfg.SPECIES_LIST or c[0] in cfg.SPECIES_LIST):
label = cfg.TRANSLATED_LABELS[cfg.LABELS.index(c[0])]
rstring += "{}\t{}\t{:.4f}\n".format(timestamp.replace("-", "\t"), label.replace("_", ", "), c[1])
ts = timestamp.replace("-", "\t")
lbl = label.replace("_", ", ")
rstring += f"{ts}\t{lbl}\t{c[1]:.4f}\n"

# Write result string to file
out_string += rstring
Expand Down Expand Up @@ -199,7 +182,9 @@ def saveResultFile(r: dict[str, list], path: str, afile_path: str):

if c[1] > cfg.MIN_CONFIDENCE and (not cfg.SPECIES_LIST or c[0] in cfg.SPECIES_LIST):
label = cfg.TRANSLATED_LABELS[cfg.LABELS.index(c[0])]
rstring += "{},{},{},{},{:.4f}\n".format(start, end, label.split("_", 1)[0], label.split("_", 1)[-1], c[1])
rstring += "{},{},{},{},{:.4f}\n".format(
start, end, label.split("_", 1)[0], label.split("_", 1)[-1], c[1]
)

# Write result string to file
out_string += rstring
Expand All @@ -208,17 +193,19 @@ def saveResultFile(r: dict[str, list], path: str, afile_path: str):
with open(path, "w", encoding="utf-8") as rfile:
rfile.write(out_string)


def combineResults(folder: str, output_file: str):

# Read all files
files = utils.collect_all_files(folder, "txt", pattern="BirdNET.selection.table")
files = utils.collect_all_files(folder, ["txt"], pattern="BirdNET.selection.table")

# Combine all files
# Combine all files
s_id = 1
time_offset = 0

audiofiles = set()

with open(os.path.join(folder, output_file), "w", encoding="utf-8") as f:
f.write("Selection\tView\tChannel\tBegin Path\tFile Duration (s)\tBegin Time (s)\tEnd Time (s)\tLow Freq (Hz)\tHigh Freq (Hz)\tSpecies Code\tCommon Name\tConfidence\n")
f.write(RTABLE_HEADER)

for rfile in files:
with open(rfile, "r", encoding="utf-8") as rf:
Expand All @@ -227,11 +214,14 @@ def combineResults(folder: str, output_file: str):
lines = rf.readlines()

# make sure it's a selection table
if not "Selection" in lines[0] or not "File Duration" in lines[0]:
if not "Selection" in lines[0] or not "File Offset" in lines[0]:
continue

# skip header and add to file
f_duration = float(lines[1].split("\t")[4])
f_name = lines[1].split("\t")[10]
f_duration = audio.getAudioFileLength(f_name, cfg.SAMPLE_RATE)

audiofiles.add(f_name)

for line in lines[1:]:

Expand All @@ -241,7 +231,7 @@ def combineResults(folder: str, output_file: str):

# Is species code and common name == 'nocall'?
# If so, that's a dummy line and we can skip it
if line.split("\t")[9] == "nocall" and line.split("\t")[10] == "nocall":
if line.split("\t")[7] == "nocall" and line.split("\t")[8] == "nocall":
continue

# adjust selection id
Expand All @@ -250,18 +240,23 @@ def combineResults(folder: str, output_file: str):
s_id += 1

# adjust time
line[5] = str(float(line[5]) + time_offset)
line[6] = str(float(line[6]) + time_offset)
line[3] = str(float(line[3]) + time_offset)
line[4] = str(float(line[4]) + time_offset)

# write line
f.write("\t".join(line))

# adjust time offset
time_offset += f_duration
time_offset += f_duration

except Exception as ex:
print(f"Error: Cannot combine results from {rfile}.\n", flush=True)
utils.writeErrorLog(ex)
utils.writeErrorLog(ex)

listfilesname = output_file.rsplit(".", 1)[0] + ".list.txt"

with open(os.path.join(folder, listfilesname), "w", encoding="utf-8") as f:
f.writelines((f + "\n" for f in audiofiles))


def getSortedTimestamps(results: dict[str, list]):
Expand Down Expand Up @@ -344,7 +339,7 @@ def analyzeFile(item):

# Process each chunk
try:
while offset < fileLengthSeconds:
while offset < fileLengthSeconds:
chunks = getRawAudioFromFile(fpath, offset, duration)
samples = []
timestamps = []
Expand Down Expand Up @@ -425,7 +420,7 @@ def analyzeFile(item):
return False

delta_time = (datetime.datetime.now() - start_time).total_seconds()
print("Finished {} in {:.2f} seconds".format(fpath, delta_time), flush=True)
print(f"Finished {fpath} in {delta_time:.2f} seconds", flush=True)

return True

Expand Down Expand Up @@ -462,10 +457,16 @@ def analyzeFile(item):
help="Detection sensitivity; Higher values result in higher sensitivity. Values in [0.5, 1.5]. Defaults to 1.0.",
)
parser.add_argument(
"--min_conf", type=float, default=0.1, help="Minimum confidence threshold. Values in [0.01, 0.99]. Defaults to 0.1."
"--min_conf",
type=float,
default=0.1,
help="Minimum confidence threshold. Values in [0.01, 0.99]. Defaults to 0.1.",
)
parser.add_argument(
"--overlap", type=float, default=0.0, help="Overlap of prediction segments. Values in [0.0, 2.9]. Defaults to 0.0."
"--overlap",
type=float,
default=0.0,
help="Overlap of prediction segments. Values in [0.0, 2.9]. Defaults to 0.0.",
)
parser.add_argument(
"--rtype",
Expand All @@ -475,9 +476,11 @@ def analyzeFile(item):
parser.add_argument(
"--output_file",
default=None,
help="Path to combined Raven selection table. If set and rtype is 'table', all results will be combined into this file. Defaults to None."
help="Path to combined Raven selection table. If set and rtype is 'table', all results will be combined into this file. Defaults to None.",
)
parser.add_argument(
"--threads", type=int, default=min(8, max(1, multiprocessing.cpu_count() // 2)), help="Number of CPU threads."
)
parser.add_argument("--threads", type=int, default=min(8, max(1, multiprocessing.cpu_count() // 2)), help="Number of CPU threads.")
parser.add_argument(
"--batchsize", type=int, default=1, help="Number of samples to process at the same time. Defaults to 1."
)
Expand All @@ -498,16 +501,16 @@ def analyzeFile(item):
help="Path to custom trained classifier. Defaults to None. If set, --lat, --lon and --locale are ignored.",
)
parser.add_argument(
"--fmin",
type=int,
default=cfg.SIG_FMIN,
help="Minimum frequency for bandpass filter in Hz. Defaults to {} Hz.".format(cfg.SIG_FMIN)
"--fmin",
type=int,
default=cfg.SIG_FMIN,
help=f"Minimum frequency for bandpass filter in Hz. Defaults to {cfg.SIG_FMIN} Hz.",
)
parser.add_argument(
"--fmax",
type=int,
default=cfg.SIG_FMAX,
help="Maximum frequency for bandpass filter in Hz. Defaults to {} Hz.".format(cfg.SIG_FMAX)
"--fmax",
type=int,
default=cfg.SIG_FMAX,
help=f"Maximum frequency for bandpass filter in Hz. Defaults to {cfg.SIG_FMAX} Hz.",
)

args = parser.parse_args()
Expand All @@ -534,7 +537,7 @@ def analyzeFile(item):
cfg.LABELS = utils.readLines(cfg.LABELS_FILE)
else:
cfg.APPLY_SIGMOID = False
cfg.LABELS_FILE = os.path.join(args.classifier, "labels", "label_names.csv")
cfg.LABELS_FILE = os.path.join(args.classifier, "labels", "label_names.csv")
cfg.LABELS = [line.split(",")[1] for line in utils.readLines(cfg.LABELS_FILE)]

args.lat = -1
Expand Down Expand Up @@ -630,19 +633,19 @@ def analyzeFile(item):
flist = [(f, cfg.getConfig()) for f in cfg.FILE_LIST]

# Analyze files
if cfg.CPU_THREADS < 2:
if cfg.CPU_THREADS < 2 or len(flist) < 2:
for entry in flist:
analyzeFile(entry)
else:
with Pool(cfg.CPU_THREADS) as p:
# Map analyzeFile function to each entry in flist
results = p.map_async(analyzeFile, flist)
# Wait for all tasks to complete
results.wait()
results.wait()

# Combine results?
if not cfg.OUTPUT_FILE is None:
print("Combining results into {}...".format(cfg.OUTPUT_FILE), end='', flush=True)
print(f"Combining results into {cfg.OUTPUT_FILE}...", end="", flush=True)
combineResults(cfg.OUTPUT_PATH, cfg.OUTPUT_FILE)
print("done!", flush=True)

Expand Down
2 changes: 1 addition & 1 deletion utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ def collect_all_files(path: str, filetypes: list[str], pattern: str = ""):

for root, _, flist in os.walk(path):
for f in flist:
if not f.startswith(".") and f.rsplit(".", 1)[-1].lower() in filetypes and (pattern in f or pattern == ""):
if not f.startswith(".") and f.rsplit(".", 1)[-1].lower() in filetypes and (pattern in f or not pattern):
files.append(os.path.join(root, f))

return sorted(files)
Expand Down

0 comments on commit 8b1da28

Please sign in to comment.