Agile Bioacoustic Modeling with SongSpace

SongSpace provides a workflow for active or “agile” learning for bioacoustics data. Embed audio into a databse, query the database with vector search or classifieres, and select clips for active learning review or final verification for ecological analyses.

Embeddings are saved in a HopLite database. The same folder storing the (sql) embedding database will also store classifiers and tables for labeled datasets. The full workspace can be saved and loaded with ss.save(path) and SongSpace.load(path).

Run this tutorial

If running in Colab, uncomment the installation line below.

# if 'google.colab' in str(get_ipython()):
#     %pip install "opensoundscape==0.12.1" "bioacoustics-model-zoo==0.12.0"

from pathlib import Path
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

import bioacoustics_model_zoo as bmz

from opensoundscape.annotations import BoxedAnnotations
from opensoundscape.vector_database import load_or_create_hoplite_usearch_db
from opensoundscape.ml.song_space import SongSpace
from opensoundscape.ml.shallow_classifier import select_from_hoplite
from opensoundscape.visualization import annotate, inspect

Prepare labels

This uses the same Rana sierrae example files as the agile Hoplite tutorial.

dataset_path = Path("./rana_sierrae_2022/")
audio_and_raven_files = pd.read_csv(dataset_path / "audio_and_raven_files.csv")
audio_and_raven_files["audio"] = audio_and_raven_files["audio"].apply(
    lambda x: str(dataset_path / x)
)
audio_and_raven_files["raven"] = audio_and_raven_files["raven"].apply(
    lambda x: str(dataset_path / x)
)

annotations = BoxedAnnotations.from_raven_files(
    raven_files=audio_and_raven_files["raven"],
    audio_files=audio_and_raven_files["audio"],
    annotation_column="annotation",
)

labels = annotations.clip_labels(clip_duration=3, min_label_overlap=0.2)

/Users/SML161/opensoundscape/opensoundscape/annotations.py:347: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.
  all_annotations_df = pd.concat(all_file_dfs).reset_index(drop=True)

target_source_class = "C"
target_model_class = "RanaSierrae_C"

# start with one recording of target class
binary_labels = labels[[target_source_class]].rename(
    columns={target_source_class: target_model_class}
)
seed_train = binary_labels.loc[
    ["rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220623_060000_0-10s.mp3"]
]
other = binary_labels.drop(seed_train.index)
validation, unlabeled = train_test_split(other, test_size=0.8, random_state=0)

print("seed_train:", seed_train.shape)
print("validation:", validation.shape)
print("pool:", unlabeled.shape)

seed_train: (4, 1)
validation: (536, 1)
pool: (2148, 1)

All audio clips from the single audio file we’ll start with for positives:

_ = inspect(seed_train, bandpass_range=(0, 2500))

import opensoundscape as opso

Build database and SongSpace

ss = SongSpace("./Perch2SongSpace", feature_extractor="perch2")

/Users/SML161/miniconda3/envs/opso_dev/lib/python3.13/site-packages/tensorflow_hub/__init__.py:61: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
  from pkg_resources import parse_version

Connecting to existing db at Perch2SongSpace
Connected database has 2,691 embeddings from 672 files.

/Users/SML161/miniconda3/envs/opso_dev/lib/python3.13/site-packages/bioacoustics_model_zoo/perch_v2.py:208: UserWarning: Disabling TensorFlow's XLA compilation (setting tf.config.optimizer.set_jit(False)) because otherwise TF models on Mac hang at runtime as of Tensorflow 2.21.0
  warnings.warn(

import opensoundscape as opso

opso.set_seed(0)

# Embed and register datasets in SongSpace.
ss.ingest_audio(
    seed_train,
    dataset_name="round1_train",
    batch_size=32,
)
ss.ingest_audio(
    validation,
    dataset_name="validation",
    allow_training=False,
    batch_size=32,
)
ss.ingest_audio(
    unlabeled,
    dataset_name="pool_unlabeled",
    batch_size=32,
)

ss.list_datasets()

all samples already have embeddings in the database
all samples already have embeddings in the database
all samples already have embeddings in the database

['round1_train', 'validation', 'pool_unlabeled']

ss.save()

Saved SongSpace to ./Perch2SongSpace with 0 classifiers and 3 datasets.

Similarity search for similar samples

# Similarity search
matches_for_each_query = ss.similarity_search(seed_train, k=20, exact_search=True)
best_matches = matches_for_each_query.sort_values(
    by="sort_score", ascending=False
).head(20)
# Review and annotate interactively.
_ = annotate(
    best_matches,
    bandpass_range=(0, 2500),
    annotation_buttons=["Accept", "Reject"],
    N=20,
)

embedding query samples

/Users/SML161/opensoundscape/opensoundscape/ml/cnn.py:2954: UserWarning: The columns of input samples df differ from `model.classes`. Discarding sample df columns.
  warnings.warn(

WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
I0000 00:00:1777756741.559128 190083460 service.cc:153] XLA service 0x398a06b40 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
I0000 00:00:1777756741.559148 190083460 service.cc:161]   StreamExecutor [0]: Host, Default Version (Driver: 0.0.0; Runtime: 0.0.0; Toolkit: 0.0.0; DNN: 0.0.0)
I0000 00:00:1777756741.809517 190083460 dump_mlir_util.cc:269] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.
W0000 00:00:1777756742.161803 190086354 cpp_gen_intrinsics.cc:74] Empty bitcode string provided for eigen. Optimizations relying on this IR will be disabled.
I0000 00:00:1777756742.162339 190086354 rsqrt.cc:179] Falling back to 1 / sqrt(x) for f32 false
I0000 00:00:1777756742.475014 190083460 device_compiler.h:208] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.

performing similarity search for each of 4 query samples

# ingest labels from the interactive labeling widget
new_pos = best_matches[best_matches["Accept"] == True][
    ["file", "start_time", "end_time"]
].copy()
new_pos[target_model_class] = 1

new_neg = best_matches[best_matches["Reject"] == True][
    ["file", "start_time", "end_time"]
].copy()
new_neg[target_model_class] = 0

search_labels = (
    pd.concat([new_pos, new_neg], ignore_index=True)
    .drop_duplicates()
    .set_index(["file", "start_time", "end_time"])[[target_model_class]]
)

# add these labels to their own dataset in the SongSpace
ss.ingest_audio(
    search_labels,
    dataset_name="search_labels",
    batch_size=32,
    num_workers=0,
)

ss.save()

search_labels[target_model_class].value_counts()

all samples already have embeddings in the database
Saved SongSpace to ./Perch2SongSpace with 0 classifiers and 4 datasets.

RanaSierrae_C
1    8
Name: count, dtype: int64

Train first classifier

clf_round1 = ss.fit_classifier(
    classes=[target_model_class],
    train_datasets=["round1_train", "search_labels"],
    validation_dataset="validation",
    weak_negatives_proportion=10.0,  # lots of weak negatives, since we have just a few positives!
    weak_negatives_weight=0.05,
    steps=100,
    batch_size=128,
    validation_interval=50,
    logging_interval=50,
)
clf_round1.val_metrics

training classifier for 1 classes with 12 training samples and 536 validation samples
Finding matching window IDs for samples in label_df...
Finding matching window IDs for samples in label_df...
Epoch 50/100, Loss: 0.031, Val Loss: 1.820
        val AU ROC: 0.803
        val MAP: 0.326
Epoch 100/100, Loss: 0.013, Val Loss: 2.213
        val AU ROC: 0.802
        val MAP: 0.332
Loaded best model with validation loss: 1.820 at step 50 of 100
Training complete

{'loss': 1.8197819471359253,
 'auroc': 0.8019947863538479,
 'map': 0.3318269957719899,
 'per_class_auroc': [0.8019947863538479]}

save the classifier in the SoundScape, if we like it enough

if "rana_round1" in ss.list_classifiers():
    ss.remove_classifier("rana_round1")
ss.add_classifier("rana_round1", clf_round1)
ss.save()

Saved SongSpace to ./Perch2SongSpace with 1 classifiers and 4 datasets.

evaluate a saved classifier on a specific dataset

round1_metrics = ss.evaluate("rana_round1", "validation")
round1_metrics

Finding matching window IDs for samples in label_df...

{'RanaSierrae_C': {'average_precision': 0.3318269957719899,
  'roc_auc': 0.8019947863538479},
 'macro_average_precision': np.float64(0.3318269957719899),
 'macro_roc_auc': np.float64(0.8019947863538479)}

Active learning round: review high-scoring candidates

pool_scores = ss.predict_on_dataset(
    classifier_name="rana_round1", dataset_name="pool_unlabeled"
)
# drop samples already labeled
labeled_idx = set(search_labels.index).union(set(seed_train.index))
pool_scores = pool_scores[~pool_scores.index.isin(labeled_idx)]
topk = pool_scores.nlargest(20, target_model_class).reset_index()

# Review and annotate interactively.
_ = annotate(
    topk, bandpass_range=(0, 2500), annotation_buttons=["Accept", "Reject"], N=20
)
topk.head()

Finding matching window IDs for samples in label_df...

	file	start_time	end_time	RanaSierrae_C	Accept	Reject
0	rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220...	0.0	3.0	4.038275	None	None
1	rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220...	3.0	6.0	4.014554	None	None
2	rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220...	6.0	9.0	3.934664	None	None
3	rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220...	6.0	9.0	3.896363	None	None
4	rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220...	6.0	9.0	3.894268	None	None

ingest labels

new_pos = topk[topk["Accept"] == True][["file", "start_time", "end_time"]].copy()
new_pos[target_model_class] = 1

new_neg = topk[topk["Reject"] == True][["file", "start_time", "end_time"]].copy()
new_neg[target_model_class] = 0

round2_train = (
    pd.concat([new_pos, new_neg], ignore_index=True)
    .drop_duplicates()
    .set_index(["file", "start_time", "end_time"])[[target_model_class]]
)

ss.ingest_audio(
    round2_train,
    dataset_name="round2_train",
    batch_size=32,
    num_workers=0,
)
ss.save()
round2_train[target_model_class].value_counts()

all samples already have embeddings in the database
Saved SongSpace to ./Perch2SongSpace with 1 classifiers and 5 datasets.

RanaSierrae_C
0    8
1    4
Name: count, dtype: int64

build a new classifier

clf_round2 = ss.fit_classifier(
    classes=[target_model_class],
    train_datasets=["round1_train", "search_labels", "round2_train"],
    validation_dataset="validation",
    weak_negatives_proportion=1.0,
    weak_negatives_weight=0.001,
    steps=200,
    batch_size=128,
    validation_interval=30,
    logging_interval=30,
)
if "rana_round2" in ss.list_classifiers():
    ss.remove_classifier("rana_round2")
ss.add_classifier("rana_round2", clf_round2)

round2_metrics = ss.evaluate("rana_round2", "validation")
round2_metrics

training classifier for 1 classes with 24 training samples and 536 validation samples
Finding matching window IDs for samples in label_df...
Finding matching window IDs for samples in label_df...
Epoch 30/200, Loss: 0.342, Val Loss: 0.629
        val AU ROC: 0.818
        val MAP: 0.575
Epoch 60/200, Loss: 0.198, Val Loss: 0.539
        val AU ROC: 0.812
        val MAP: 0.558
Epoch 90/200, Loss: 0.131, Val Loss: 0.494
        val AU ROC: 0.805
        val MAP: 0.563
Epoch 120/200, Loss: 0.094, Val Loss: 0.469
        val AU ROC: 0.801
        val MAP: 0.560
Epoch 150/200, Loss: 0.071, Val Loss: 0.454
        val AU ROC: 0.796
        val MAP: 0.555
Epoch 180/200, Loss: 0.056, Val Loss: 0.444
        val AU ROC: 0.793
        val MAP: 0.548
Loaded best model with validation loss: 0.444 at step 180 of 200
Training complete
Finding matching window IDs for samples in label_df...

{'RanaSierrae_C': {'average_precision': 0.5429201413929045,
  'roc_auc': 0.7913408137821604},
 'macro_average_precision': np.float64(0.5429201413929045),
 'macro_roc_auc': np.float64(0.7913408137821604)}

Active learning round 2: review high-scoring candidates

pool_scores = ss.predict_on_dataset(
    classifier_name="rana_round2", dataset_name="pool_unlabeled"
)
# drop samples already labeled
labeled_idx = (
    set(search_labels.index).union(set(seed_train.index)).union(set(round2_train.index))
)
pool_scores = pool_scores[~pool_scores.index.isin(labeled_idx)]
topk = pool_scores.nlargest(20, target_model_class).reset_index()

# Review and annotate interactively.
_ = annotate(
    topk, bandpass_range=(0, 2500), annotation_buttons=["Accept", "Reject"], N=20
)
topk.head()

Finding matching window IDs for samples in label_df...

	file	start_time	end_time	RanaSierrae_C	Accept	Reject
0	rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220...	6.0	9.0	5.725680	None	None
1	rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220...	3.0	6.0	5.206980	None	None
2	rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220...	0.0	3.0	4.048398	None	None
3	rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220...	3.0	6.0	3.675750	None	None
4	rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220...	9.0	12.0	3.413406	None	None

ingest labels

new_pos = topk[topk["Accept"] == True][["file", "start_time", "end_time"]].copy()
new_pos[target_model_class] = 1

new_neg = topk[topk["Reject"] == True][["file", "start_time", "end_time"]].copy()
new_neg[target_model_class] = 0

round3_train = (
    pd.concat([new_pos, new_neg], ignore_index=True)
    .drop_duplicates()
    .set_index(["file", "start_time", "end_time"])[[target_model_class]]
)

ss.ingest_audio(
    round3_train,
    dataset_name="round3_train",
    batch_size=32,
    num_workers=0,
)
ss.save()
round3_train[target_model_class].value_counts()

Saved SongSpace to ./Perch2SongSpace with 2 classifiers and 6 datasets.

Series([], Name: count, dtype: int64)

build new classifier

clf_round3 = ss.fit_classifier(
    classes=[target_model_class],
    train_datasets=["round1_train", "search_labels", "round2_train", "round3_train"],
    validation_dataset="validation",
    weak_negatives_proportion=1.0,
    weak_negatives_weight=0.001,
    steps=200,
    batch_size=128,
    validation_interval=30,
    logging_interval=30,
)
if "rana_round3" in ss.list_classifiers():
    ss.remove_classifier("rana_round3")
ss.add_classifier("rana_round3", clf_round3)

ss.save()

round3_metrics = ss.evaluate("rana_round3", "validation")
round3_metrics

training classifier for 1 classes with 24 training samples and 536 validation samples
Finding matching window IDs for samples in label_df...
Finding matching window IDs for samples in label_df...
Epoch 30/200, Loss: 0.349, Val Loss: 0.644
        val AU ROC: 0.818
        val MAP: 0.557
Epoch 60/200, Loss: 0.203, Val Loss: 0.553
        val AU ROC: 0.811
        val MAP: 0.549
Epoch 90/200, Loss: 0.134, Val Loss: 0.506
        val AU ROC: 0.804
        val MAP: 0.553
Epoch 120/200, Loss: 0.096, Val Loss: 0.481
        val AU ROC: 0.799
        val MAP: 0.551
Epoch 150/200, Loss: 0.073, Val Loss: 0.466
        val AU ROC: 0.793
        val MAP: 0.540
Epoch 180/200, Loss: 0.058, Val Loss: 0.456
        val AU ROC: 0.790
        val MAP: 0.528
Loaded best model with validation loss: 0.456 at step 180 of 200
Training complete
Saved SongSpace to ./Perch2SongSpace with 3 classifiers and 6 datasets.
Finding matching window IDs for samples in label_df...

{'RanaSierrae_C': {'average_precision': 0.5286647807313382,
  'roc_auc': 0.7881672900374023},
 'macro_average_precision': np.float64(0.5286647807313382),
 'macro_roc_auc': np.float64(0.7881672900374023)}

we now have a solid classifier to use for downstream tasks.

Select clips for manual verification

Use stratified or thresholded selection from the full embedded database.

select_from_hoplite provides several options for filtering. We can loop over the variables of interest to select stratified clips.

Filtering options include:

• first and last date

• earliest and latest time

• minimum and maximum score

• list of recordings (audio file paths)

• list of deployments

• list of projects

We also specify which classes we want to extract clips for, how many, and under which strategy:

• top_k: highest scoring k (eg, 5) clips matching the filters

• random_k: randomly selected k clips matching the filters

• all: all clips matching the filters

# select the global 5 most confident 'RanaSierrae_C' clips from the pool according to the round 3 classifier
clips = select_from_hoplite(
    db=ss.db,
    classifier=ss.classifiers["rana_round3"],
    classes=["RanaSierrae_C"],
    strategy="top_k",
    k=5,
)
inspect(clips, bandpass_range=(0, 2500))
clips

	file	start_time	end_time	datetime	deployment	project	window_id	class
0	rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220...	3.0	6.0	2022-06-23 06:15:00	mp3	round1_train	1264	RanaSierrae_C
1	rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220...	3.0	6.0	2022-06-22 20:15:00	mp3	round1_train	1220	RanaSierrae_C
2	rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220...	0.0	3.0	2022-06-22 18:15:00	mp3	round1_train	599	RanaSierrae_C
3	rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220...	6.0	9.0	2022-06-22 20:15:00	mp3	round1_train	1219	RanaSierrae_C
4	rana_sierrae_2022/mp3/sine2022a_MSD-0558_20220...	6.0	9.0	2022-06-23 06:00:00	mp3	round1_train	3	RanaSierrae_C

We can re-load the SongSpace in another Python session, which will retain all the saved classifiers, labeled datasets, and embeddings. The clip dataframes created by these examples can be saved to CVS for annotation in Dipper or other review software.

# reload the SongSpace, as we would in a new script/notebook
from opensoundscape import SongSpace
from opensoundscape.visualization import inspect

ss_reloaded = SongSpace.open("./Perch2SongSpace")
print(f"Classifiers: {ss_reloaded.list_classifiers()}")
print(f"Datasets: {ss_reloaded.list_datasets()}")

Connecting to existing db at Perch2SongSpace
Connected database has 2,691 embeddings from 672 files.
Classifiers: ['rana_round1', 'rana_round2', 'rana_round3']
Datasets: ['round1_train', 'validation', 'pool_unlabeled', 'search_labels', 'round2_train', 'round3_train']

/Users/SML161/miniconda3/envs/opso_dev/lib/python3.13/site-packages/bioacoustics_model_zoo/perch_v2.py:208: UserWarning: Disabling TensorFlow's XLA compilation (setting tf.config.optimizer.set_jit(False)) because otherwise TF models on Mac hang at runtime as of Tensorflow 2.21.0
  warnings.warn(

Example: Stratify by date range and deployment

a typical stratification pattern for reviewing clips for an occupancy analysis

date_ranges = [
    ("2022-06-20", "2022-06-21"),
    ("2022-06-22", "2022-06-23"),
    ("2022-06-24", "2022-06-25"),
    ("2022-06-26", "2022-06-27"),
]
clips = ss_reloaded.stratified_selection(
    ss_reloaded.classifiers["rana_round3"],
    classes=["RanaSierrae_C"],
    stratify_deployments=True,
    k=1,
    date_ranges=date_ranges,
)


# table ready for Dipper review with stratification by "date_range" and "deployment" in binary annotation mode
# selected.to_csv('RanaSierrae_C_clips_for_review.csv')

Other stratification and filteringpatterns

Let’s now select the highest scorking global k=2 clips for each of 4 date ranges. We’ll enforce a score threshold of 0.

selected = ss_reloaded.stratified_selection(
    classifier=ss_reloaded.classifiers["rana_round3"],
    classes=["RanaSierrae_C"],
    strategy="top_k",
    k=2,
    min_score=0,
    date_ranges=date_ranges,
)
for date_range, clips in selected.groupby("date_range"):
    print(f"Date range: {date_range}")
    inspect(selected, bandpass_range=(0, 2500))

Date range: 2022-06-20 to 2022-06-21

Date range: 2022-06-22 to 2022-06-23