clap2-live-taxonomy / engine.py
ChristophSchuhmann's picture
Add 100M caption index choices and ten-minute batched sliding-window analysis
d747083 verified
Raw History Blame Contribute Delete
27 kB
"""CLAP2-backed audio timeline engine for the public taxonomy demo."""
from __future__ import annotations
import bisect
import json
import math
import os
import subprocess
import tempfile
from contextlib import nullcontext
from pathlib import Path
from typing import Any
import numpy as np
SAMPLE_RATE = 32_000
EMBEDDING_DIM = 640
CLASS_COUNT = 4_416
MODEL_REPOSITORY = "laion/clap2-medium-audioxl-s1B"
MODEL_REVISION = "8e299e83c08a291711490105e82a345abadb5cdb"
def _finite_float(value: Any, name: str) -> float:
try:
result = float(value)
except (TypeError, ValueError) as exc:
raise ValueError(f"{name} must be a finite number") from exc
if not math.isfinite(result):
raise ValueError(f"{name} must be a finite number")
return result
def load_audio(path: str, max_seconds: float = 600) -> tuple[np.ndarray, int]:
"""Decode a complete mono waveform at 32 kHz with ffmpeg.
Accepted uploads are decoded through EOF, so duration does not depend on
container duration estimates (which can truncate VBR MP3 files). For an
overlong input, decoding stops as soon as more than ``max_seconds`` worth
of samples arrives, then rejects the file explicitly.
"""
source = Path(path)
if not source.is_file():
raise FileNotFoundError(f"Audio file does not exist: {source}")
limit = _finite_float(max_seconds, "max_seconds")
if limit <= 0:
raise ValueError("max_seconds must be greater than zero")
if limit > 600.0:
raise ValueError("max_seconds cannot exceed the 600-second upload limit")
max_samples = int(math.floor(limit * SAMPLE_RATE + 1e-9))
command = [
"ffmpeg",
"-hide_banner",
"-loglevel",
"error",
"-nostdin",
"-protocol_whitelist",
"file,pipe",
"-i",
str(source),
"-map",
"0:a:0",
"-vn",
"-sn",
"-dn",
"-ac",
"1",
"-ar",
str(SAMPLE_RATE),
"-acodec",
"pcm_f32le",
"-f",
"f32le",
"pipe:1",
]
stderr_file = tempfile.TemporaryFile()
process: subprocess.Popen[bytes] | None = None
chunks: list[bytes] = []
total_bytes = 0
over_limit = False
try:
process = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=stderr_file,
bufsize=0,
)
assert process.stdout is not None
while True:
block = process.stdout.read(65_536)
if not block:
break
chunks.append(block)
total_bytes += len(block)
# Reads may end within a sample; use floor so a partial trailing
# sample cannot cause a false over-limit rejection.
if total_bytes // 4 > max_samples:
over_limit = True
process.kill()
break
process.stdout.close()
return_code = process.wait()
if over_limit:
raise ValueError(f"Audio exceeds the {limit:g}-second limit")
if return_code != 0:
stderr_file.seek(0)
diagnostic = stderr_file.read().decode("utf-8", errors="replace").strip()
raise RuntimeError(f"ffmpeg could not decode audio: {diagnostic or 'unknown error'}")
if total_bytes % 4:
raise RuntimeError("ffmpeg returned an incomplete float32 audio sample")
pcm = np.frombuffer(b"".join(chunks), dtype="<f4").astype(np.float32, copy=True)
if pcm.size == 0:
raise ValueError("Audio contains no decodable samples")
if not np.isfinite(pcm).all():
raise ValueError("Decoded audio contains non-finite samples")
if pcm.size > max_samples:
raise ValueError(f"Audio exceeds the {limit:g}-second limit")
return pcm, SAMPLE_RATE
except BaseException:
if process is not None and process.poll() is None:
if process.stdout is not None:
try:
process.stdout.close()
except OSError:
pass
try:
process.kill()
except OSError:
pass
process.wait()
raise
finally:
stderr_file.close()
def _read_nodes(path: Path) -> list[dict[str, Any]]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except FileNotFoundError:
raise
except (json.JSONDecodeError, OSError) as exc:
raise ValueError(f"Could not read node metadata from {path}") from exc
rows = payload.get("nodes") if isinstance(payload, dict) else payload
if not isinstance(rows, list) or len(rows) != CLASS_COUNT:
raise ValueError(f"{path.name} must contain exactly {CLASS_COUNT} nodes")
if any(not isinstance(row, dict) for row in rows):
raise ValueError(f"{path.name} contains a non-object node")
ids = [str(row.get("id", "")) for row in rows]
if any(not node_id for node_id in ids) or len(set(ids)) != CLASS_COUNT:
raise ValueError(f"{path.name} contains missing or duplicate node IDs")
return rows
def _path_text(node: dict[str, Any]) -> str:
value = node.get("path_text")
if isinstance(value, str) and value.strip():
return value.strip()
value = node.get("path", "")
if isinstance(value, (list, tuple)):
return " › ".join(str(part).strip() for part in value if str(part).strip())
return str(value or "").strip()
def _normalize_embeddings(embeddings: np.ndarray, name: str) -> np.ndarray:
matrix = np.asarray(embeddings, dtype=np.float32)
if matrix.shape != (CLASS_COUNT, EMBEDDING_DIM):
raise ValueError(
f"{name} embeddings must have shape ({CLASS_COUNT}, {EMBEDDING_DIM}); got {matrix.shape}"
)
if not np.isfinite(matrix).all():
raise ValueError(f"{name} embeddings contain non-finite values")
norms = np.linalg.norm(matrix, axis=1, keepdims=True)
if np.any(norms <= 1e-12) or not np.isfinite(norms).all():
raise ValueError(f"{name} embeddings contain zero-length vectors")
return np.ascontiguousarray(matrix / norms, dtype=np.float32)
def _load_index(path: Path, kind: str) -> tuple[np.ndarray, list[str], list[str]]:
try:
with np.load(path, allow_pickle=False) as archive:
required = {"embeddings", "node_ids", "prompts"}
missing = required.difference(archive.files)
if missing:
raise ValueError(f"{path.name} is missing arrays: {sorted(missing)}")
embeddings = np.array(archive["embeddings"], dtype=np.float32, copy=True)
ids = [str(value) for value in archive["node_ids"].tolist()]
prompts = [str(value) for value in archive["prompts"].tolist()]
except FileNotFoundError:
raise
except (OSError, ValueError) as exc:
raise ValueError(f"Could not read {kind} index at {path}") from exc
if len(ids) != CLASS_COUNT or len(set(ids)) != CLASS_COUNT:
raise ValueError(f"{path.name} must contain {CLASS_COUNT} unique node IDs")
if len(prompts) != CLASS_COUNT:
raise ValueError(f"{path.name} must contain {CLASS_COUNT} prompts")
return _normalize_embeddings(embeddings, kind), ids, prompts
def _time_edges(duration: float, stride: float) -> np.ndarray:
"""Make stride-spaced edges and append the exact duration as the tail."""
full_steps = int(math.floor(duration / stride + 1e-10))
edges = np.arange(full_steps + 1, dtype=np.float64) * stride
if edges.size == 0:
edges = np.array([0.0], dtype=np.float64)
if edges[-1] >= duration - max(1e-10, duration * 1e-12):
edges[-1] = duration
else:
edges = np.append(edges, duration)
if edges.size < 2:
edges = np.array([0.0, duration], dtype=np.float64)
return edges
def _validate_pcm(pcm: Any) -> tuple[np.ndarray, float]:
waveform = np.asarray(pcm, dtype=np.float32)
if waveform.ndim != 1:
raise ValueError("pcm must be a one-dimensional mono waveform")
if waveform.size == 0:
raise ValueError("pcm must contain at least one sample")
if not np.isfinite(waveform).all():
raise ValueError("pcm contains non-finite samples")
duration = waveform.size / SAMPLE_RATE
if duration > 600.0:
raise ValueError("Audio exceeds the 600-second limit")
return np.ascontiguousarray(waveform), duration
class AudioEngine:
"""Load CLAP2 indexes and encode synchronized audio timeline bins."""
def __init__(self, data_dir: str, device: str = "cuda", cache_dir: str | None = None):
self.data_dir = Path(data_dir)
self.device = str(device)
self.cache_dir = cache_dir
original_embeddings, original_ids, self.original_prompts = _load_index(
self.data_dir / "original-index.npz", "original"
)
sentence_embeddings, sentence_ids, self.sentence_prompts = _load_index(
self.data_dir / "sentence-index.npz", "sentence"
)
original_rows = _read_nodes(self.data_dir / "original-nodes.json")
sentence_rows = _read_nodes(self.data_dir / "sentence-nodes.json")
if len(set(original_ids)) != CLASS_COUNT or set(original_ids) != set(sentence_ids):
raise ValueError("Original and sentence index node IDs must match exactly")
# The original index defines the canonical class order. Reorder the
# sentence matrix if its archive was serialized in another row order.
sentence_row_by_id = {node_id: i for i, node_id in enumerate(sentence_ids)}
sentence_reorder = np.asarray([sentence_row_by_id[node_id] for node_id in original_ids])
sentence_embeddings = np.ascontiguousarray(sentence_embeddings[sentence_reorder])
self.sentence_prompts = [self.sentence_prompts[i] for i in sentence_reorder]
original_by_id = {str(row["id"]): row for row in original_rows}
sentence_by_id = {str(row["id"]): row for row in sentence_rows}
if set(original_ids) != set(original_by_id) or set(original_ids) != set(sentence_by_id):
raise ValueError("Index node IDs must match their corresponding node metadata")
nodes: list[dict[str, str]] = []
for node_id in original_ids:
original = original_by_id[node_id]
sentence_node = sentence_by_id[node_id]
label = str(original.get("label", "")).strip()
sentence = str(sentence_node.get("sentence", "")).strip()
if not label or not sentence:
raise ValueError(f"Node {node_id} is missing a label or sentence")
path = _path_text(original) or _path_text(sentence_node)
nodes.append({"id": node_id, "label": label, "path": path, "sentence": sentence})
self.nodes = nodes
self.original_text = original_embeddings
self.sentence_text = sentence_embeddings
# Import model dependencies only here, so pure utilities stay usable
# without downloading weights or initializing CUDA.
import torch
import open_clip
from huggingface_hub import snapshot_download
torch.set_num_threads(2)
if self.device.startswith("cuda") and torch.cuda.is_available():
torch.backends.cuda.matmul.allow_tf32 = True
model_dir = snapshot_download(
repo_id=MODEL_REPOSITORY,
revision=MODEL_REVISION,
allow_patterns=["open_clip*"],
cache_dir=cache_dir,
)
model, preprocess_factory = open_clip.create_model_from_pretrained(
f"local-dir:{model_dir}", device=self.device
)
self.model = model.eval().to(self.device)
self.preprocess = preprocess_factory()
self._torch = torch
def _amp_context(self):
if self.device.startswith("cuda"):
return self._torch.autocast(device_type="cuda", dtype=self._torch.bfloat16)
return nullcontext()
def _preprocess_windows(self, pcm: np.ndarray, starts: list[int], width: int) -> np.ndarray:
torch = self._torch
prepared: list[dict[str, Any]] = []
vectors: list[np.ndarray] = []
window_count = len(starts)
for batch_start in range(0, window_count, 64):
batch_starts = starts[batch_start : batch_start + 64]
prepared.clear()
for start in batch_starts:
end = min(start + width, pcm.size)
crop = torch.from_numpy(pcm[start:end].copy())
prepared.append(self.preprocess((crop, SAMPLE_RATE)))
if not prepared:
continue
keys = prepared[0].keys()
if any(item.keys() != prepared[0].keys() for item in prepared):
raise ValueError("CLAP2 audio preprocessing returned inconsistent fields")
batch = {
key: torch.stack([item[key] for item in prepared]).to(self.device)
for key in keys
}
with torch.inference_mode(), self._amp_context():
encoded = self.model.encode_audio(batch, normalize=True)
encoded = encoded.float()
encoded = torch.nn.functional.normalize(encoded, dim=-1)
array = encoded.cpu().numpy().astype(np.float32, copy=False)
if array.shape != (len(batch_starts), EMBEDDING_DIM):
raise ValueError(f"CLAP2 returned unexpected audio embedding shape {array.shape}")
if not np.isfinite(array).all():
raise ValueError("CLAP2 returned non-finite audio embeddings")
vectors.append(array)
if not vectors:
raise ValueError("No audio windows were produced")
return np.concatenate(vectors, axis=0)
def _encode_sliding(self, pcm: np.ndarray, duration: float, window_seconds: float, stride_seconds: float):
window = min(window_seconds, duration)
width = max(1, int(round(window * SAMPLE_RATE)))
hop = max(1, int(round(stride_seconds * SAMPLE_RATE)))
if pcm.size <= width:
starts = [0]
else:
last_start = pcm.size - width
starts = list(range(0, last_start + 1, hop))
if starts[-1] != last_start:
starts.append(last_start)
ends = [min(start + width, pcm.size) for start in starts]
window_embeddings = self._preprocess_windows(pcm, starts, width)
edges = _time_edges(duration, stride_seconds)
output = np.zeros((edges.size - 1, EMBEDDING_DIM), dtype=np.float64)
weight_sums = np.zeros(edges.size - 1, dtype=np.float64)
win_starts_s = np.asarray(starts, dtype=np.float64) / SAMPLE_RATE
win_ends_s = np.asarray(ends, dtype=np.float64) / SAMPLE_RATE
for bin_i, (bin_start, bin_end) in enumerate(zip(edges[:-1], edges[1:])):
overlap = np.maximum(
0.0,
np.minimum(win_ends_s, bin_end) - np.maximum(win_starts_s, bin_start),
)
selected = overlap > 1e-12
if not selected.any():
raise RuntimeError(f"No audio windows overlap time bin {bin_i}")
weights = overlap[selected]
output[bin_i] = (window_embeddings[selected].astype(np.float64) * weights[:, None]).sum(axis=0)
weight_sums[bin_i] = weights.sum()
output /= weight_sums[:, None]
if not np.isfinite(output).all():
raise ValueError("Audio embedding aggregation produced non-finite values")
return output.astype(np.float32), edges, len(starts)
def _encode_contextual(self, pcm: np.ndarray, duration: float, stride_seconds: float):
if duration > 30.0:
raise ValueError("Contextual patches support clips up to 30 seconds")
torch = self._torch
import torch.nn.functional as F
from torch.nn.utils.rnn import pad_sequence
tower = self.model.audio
vit = tower.encoder.vit
if getattr(vit, "pool_include_prefix", False) or vit.attn_pool.pos_embed is not None:
raise RuntimeError("CLAP2 contextual pooling requires prefix exclusion and no pool position embedding")
edges = _time_edges(duration, stride_seconds)
waveform = torch.from_numpy(pcm.copy())
item = self.preprocess((waveform, SAMPLE_RATE))
if "patch_coord" not in item:
raise RuntimeError("CLAP2 preprocessing did not provide patch_coord")
coords = item["patch_coord"]
if isinstance(coords, torch.Tensor):
coords_np = coords.detach().cpu().numpy()
else:
coords_np = np.asarray(coords)
if coords_np.ndim != 2 or coords_np.shape[1] < 2:
raise RuntimeError("CLAP2 patch_coord has an unexpected shape")
batch = {
key: pad_sequence([value], batch_first=True).to(self.device)
for key, value in item.items()
}
with torch.inference_mode(), self._amp_context():
features = vit.forward_features(**batch)
patches = features["patches"]
time_index = coords_np[:, 1].astype(np.float64)
patch_starts = time_index * 0.16
patch_ends = np.minimum((time_index + 1.0) * 0.16, duration)
real_durations = np.maximum(patch_ends - patch_starts, 1e-9)
hidden = patches[0, vit.num_prefix_tokens : vit.num_prefix_tokens + len(coords_np)]
selected_hidden = []
selected_priors = []
for bin_start, bin_end in zip(edges[:-1], edges[1:]):
overlap = np.maximum(
0.0,
np.minimum(patch_ends, bin_end) - np.maximum(patch_starts, bin_start),
)
indices = np.flatnonzero(overlap > 1e-8)
if indices.size == 0:
raise RuntimeError(
f"No CLAP2 audio patches overlap time bin [{bin_start:.3f}, {bin_end:.3f}]"
)
selected_hidden.append(hidden[torch.as_tensor(indices, device=hidden.device)])
prior = np.log(overlap[indices] / real_durations[indices])
selected_priors.append(
torch.as_tensor(prior, device=hidden.device, dtype=hidden.dtype)
)
x = pad_sequence(selected_hidden, batch_first=True)
mask = pad_sequence(
selected_priors, batch_first=True, padding_value=float("-inf")
)[:, None, None, :]
pooled = vit.attn_pool(x, attn_mask=mask)
pooled = vit.head(vit.head_drop(vit.fc_norm(pooled)))
if getattr(tower, "pre_norm", False):
pooled = F.normalize(pooled, dim=-1)
embeddings = F.normalize(tower.proj(pooled).float(), dim=-1)
result = embeddings.detach().cpu().numpy().astype(np.float32, copy=False)
if result.shape != (edges.size - 1, EMBEDDING_DIM):
raise ValueError(f"CLAP2 contextual pooling returned unexpected shape {result.shape}")
if not np.isfinite(result).all():
raise ValueError("CLAP2 contextual pooling returned non-finite embeddings")
return result, edges
def encode(
self,
pcm: np.ndarray,
window_seconds: float = 2.0,
stride_seconds: float = 0.2,
analysis_method: str = "Sliding windows",
) -> dict[str, Any]:
waveform, duration = _validate_pcm(pcm)
window = _finite_float(window_seconds, "window_seconds")
stride = _finite_float(stride_seconds, "stride_seconds")
if not 0.1 <= stride <= 1.0:
raise ValueError("stride_seconds must be between 0.1 and 1.0")
method = str(analysis_method).strip()
if method not in {"Sliding windows", "Contextual patches"}:
raise ValueError("analysis_method must be 'Sliding windows' or 'Contextual patches'")
if method == "Sliding windows":
if not 0.5 <= window <= 10.0:
raise ValueError("window_seconds must be between 0.5 and 10.0")
if stride > window:
raise ValueError("stride_seconds cannot exceed window_seconds")
embeddings, edges, window_count = self._encode_sliding(waveform, duration, window, stride)
else:
if duration > 30.0:
raise ValueError("Contextual patches support clips up to 30 seconds")
embeddings, edges = self._encode_contextual(waveform, duration, stride)
window_count = 1
return {
"embeddings": np.ascontiguousarray(embeddings, dtype=np.float32),
"bin_edges": np.asarray(edges, dtype=np.float64),
"duration_seconds": float(duration),
"window_count": int(window_count),
"window_seconds": float(window),
"stride_seconds": float(stride),
"analysis_method": method,
}
def smooth_scores(scores: Any, bin_edges: Any, method: str, width: float) -> np.ndarray:
"""Smooth per-bin class scores using duration-aware sparse weights."""
matrix = np.asarray(scores)
if matrix.ndim != 2 or matrix.shape[0] == 0 or matrix.shape[1] == 0:
raise ValueError("scores must be a nonempty two-dimensional matrix")
if not np.isfinite(matrix).all():
raise ValueError("scores contain non-finite values")
edges = np.asarray(bin_edges, dtype=np.float64)
if edges.ndim != 1 or edges.size != matrix.shape[0] + 1:
raise ValueError("bin_edges must have length N + 1")
if not np.isfinite(edges).all() or np.any(np.diff(edges) <= 0):
raise ValueError("bin_edges must be finite and strictly increasing")
normalized_method = str(method).strip().lower()
if normalized_method in {"none", ""}:
return np.array(matrix, copy=True)
if normalized_method not in {"moving average", "gaussian"}:
raise ValueError("method must be None, Moving average, or Gaussian")
width_seconds = _finite_float(width, "width")
if width_seconds <= 0:
raise ValueError("width must be positive for score smoothing")
from scipy.sparse import csr_matrix
starts = edges[:-1]
ends = edges[1:]
centers = (starts + ends) * 0.5
durations = ends - starts
rows: list[int] = []
columns: list[int] = []
values: list[float] = []
if normalized_method == "gaussian":
sigma = width_seconds / 2.35482
radius = 3.0 * sigma
else:
radius = width_seconds * 0.5
for i, center in enumerate(centers):
left = center - radius
right = center + radius
first = max(0, int(np.searchsorted(ends, left, side="right")))
last = min(len(centers), int(np.searchsorted(starts, right, side="left")))
js = np.arange(first, max(first, last), dtype=np.int64)
if normalized_method == "moving average":
weights = np.maximum(
0.0,
np.minimum(ends[js], right) - np.maximum(starts[js], left),
)
else:
weights = durations[js] * np.exp(-0.5 * ((centers[js] - center) / sigma) ** 2)
keep = weights > 0
js = js[keep]
weights = weights[keep]
total = float(weights.sum())
if total <= 0:
js = np.array([i], dtype=np.int64)
weights = np.array([1.0], dtype=np.float64)
total = 1.0
rows.extend([i] * len(js))
columns.extend(js.tolist())
values.extend((weights / total).tolist())
weight_matrix = csr_matrix((values, (rows, columns)), shape=(len(centers), len(centers)))
result = weight_matrix @ matrix.astype(np.float64, copy=False)
return np.asarray(result, dtype=matrix.dtype if np.issubdtype(matrix.dtype, np.floating) else np.float32)
def make_payload(
nodes: list[dict[str, Any]],
scores_original: Any,
scores_sentences: Any,
bin_edges: Any,
settings: dict[str, Any],
audio_url: str,
) -> dict[str, Any]:
"""Build top-five retrieval rows with stable score-tie ordering."""
edge_array = np.asarray(bin_edges, dtype=np.float64)
if edge_array.ndim != 1 or edge_array.size < 2 or not np.isfinite(edge_array).all():
raise ValueError("bin_edges must be a finite one-dimensional array")
if np.any(np.diff(edge_array) <= 0):
raise ValueError("bin_edges must be strictly increasing")
count = edge_array.size - 1
if len(nodes) != CLASS_COUNT:
raise ValueError(f"nodes must contain exactly {CLASS_COUNT} classes")
original = np.asarray(scores_original)
sentence = np.asarray(scores_sentences)
expected = (count, CLASS_COUNT)
if original.shape != expected or sentence.shape != expected:
raise ValueError(f"score arrays must both have shape {expected}")
if not np.isfinite(original).all() or not np.isfinite(sentence).all():
raise ValueError("score arrays contain non-finite values")
ids = [str(node.get("id", "")) for node in nodes]
if any(not node_id for node_id in ids) or len(set(ids)) != CLASS_COUNT:
raise ValueError("nodes must have unique non-empty IDs")
if any(not str(node.get("label", "")).strip() for node in nodes):
raise ValueError("each node must have a label")
if any(not str(node.get("sentence", "")).strip() for node in nodes):
raise ValueError("each node must have a sentence")
class_indices = np.arange(CLASS_COUNT, dtype=np.int64)
top_original: list[list[dict[str, Any]]] = []
top_sentences: list[list[dict[str, Any]]] = []
for bin_i in range(count):
# lexsort's last key is primary: descending score, then ascending
# taxonomy index for exact ties; it never depends on hash/random state.
order_original = np.lexsort((class_indices, -original[bin_i]))[:5]
order_sentence = np.lexsort((class_indices, -sentence[bin_i]))[:5]
top_original.append(
[
{
"id": ids[j],
"label": str(nodes[j]["label"]),
"taxonomy_path": nodes[j].get("path", ""),
"score": float(original[bin_i, j]),
}
for j in order_original
]
)
top_sentences.append(
[
{
"id": ids[j],
"label": str(nodes[j]["label"]),
"sentence": str(nodes[j]["sentence"]),
"taxonomy_path": nodes[j].get("path", ""),
"score": float(sentence[bin_i, j]),
}
for j in order_sentence
]
)
return {
"audio_url": str(audio_url),
"duration_seconds": float(edge_array[-1] - edge_array[0]),
"bin_edges": edge_array.tolist(),
"original_top5": top_original,
"sentences_top5": top_sentences,
"settings": dict(settings),
}