"""CLAP2-backed audio timeline engine for the public taxonomy demo.""" from __future__ import annotations import bisect import json import math import os import subprocess import tempfile from contextlib import nullcontext from pathlib import Path from typing import Any import numpy as np SAMPLE_RATE = 32_000 EMBEDDING_DIM = 640 CLASS_COUNT = 4_416 MODEL_REPOSITORY = "laion/clap2-medium-audioxl-s1B" MODEL_REVISION = "8e299e83c08a291711490105e82a345abadb5cdb" def _finite_float(value: Any, name: str) -> float: try: result = float(value) except (TypeError, ValueError) as exc: raise ValueError(f"{name} must be a finite number") from exc if not math.isfinite(result): raise ValueError(f"{name} must be a finite number") return result def load_audio(path: str, max_seconds: float = 600) -> tuple[np.ndarray, int]: """Decode a complete mono waveform at 32 kHz with ffmpeg. Accepted uploads are decoded through EOF, so duration does not depend on container duration estimates (which can truncate VBR MP3 files). For an overlong input, decoding stops as soon as more than ``max_seconds`` worth of samples arrives, then rejects the file explicitly. """ source = Path(path) if not source.is_file(): raise FileNotFoundError(f"Audio file does not exist: {source}") limit = _finite_float(max_seconds, "max_seconds") if limit <= 0: raise ValueError("max_seconds must be greater than zero") if limit > 600.0: raise ValueError("max_seconds cannot exceed the 600-second upload limit") max_samples = int(math.floor(limit * SAMPLE_RATE + 1e-9)) command = [ "ffmpeg", "-hide_banner", "-loglevel", "error", "-nostdin", "-protocol_whitelist", "file,pipe", "-i", str(source), "-map", "0:a:0", "-vn", "-sn", "-dn", "-ac", "1", "-ar", str(SAMPLE_RATE), "-acodec", "pcm_f32le", "-f", "f32le", "pipe:1", ] stderr_file = tempfile.TemporaryFile() process: subprocess.Popen[bytes] | None = None chunks: list[bytes] = [] total_bytes = 0 over_limit = False try: process = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=stderr_file, bufsize=0, ) assert process.stdout is not None while True: block = process.stdout.read(65_536) if not block: break chunks.append(block) total_bytes += len(block) # Reads may end within a sample; use floor so a partial trailing # sample cannot cause a false over-limit rejection. if total_bytes // 4 > max_samples: over_limit = True process.kill() break process.stdout.close() return_code = process.wait() if over_limit: raise ValueError(f"Audio exceeds the {limit:g}-second limit") if return_code != 0: stderr_file.seek(0) diagnostic = stderr_file.read().decode("utf-8", errors="replace").strip() raise RuntimeError(f"ffmpeg could not decode audio: {diagnostic or 'unknown error'}") if total_bytes % 4: raise RuntimeError("ffmpeg returned an incomplete float32 audio sample") pcm = np.frombuffer(b"".join(chunks), dtype=" max_samples: raise ValueError(f"Audio exceeds the {limit:g}-second limit") return pcm, SAMPLE_RATE except BaseException: if process is not None and process.poll() is None: if process.stdout is not None: try: process.stdout.close() except OSError: pass try: process.kill() except OSError: pass process.wait() raise finally: stderr_file.close() def _read_nodes(path: Path) -> list[dict[str, Any]]: try: payload = json.loads(path.read_text(encoding="utf-8")) except FileNotFoundError: raise except (json.JSONDecodeError, OSError) as exc: raise ValueError(f"Could not read node metadata from {path}") from exc rows = payload.get("nodes") if isinstance(payload, dict) else payload if not isinstance(rows, list) or len(rows) != CLASS_COUNT: raise ValueError(f"{path.name} must contain exactly {CLASS_COUNT} nodes") if any(not isinstance(row, dict) for row in rows): raise ValueError(f"{path.name} contains a non-object node") ids = [str(row.get("id", "")) for row in rows] if any(not node_id for node_id in ids) or len(set(ids)) != CLASS_COUNT: raise ValueError(f"{path.name} contains missing or duplicate node IDs") return rows def _path_text(node: dict[str, Any]) -> str: value = node.get("path_text") if isinstance(value, str) and value.strip(): return value.strip() value = node.get("path", "") if isinstance(value, (list, tuple)): return " › ".join(str(part).strip() for part in value if str(part).strip()) return str(value or "").strip() def _normalize_embeddings(embeddings: np.ndarray, name: str) -> np.ndarray: matrix = np.asarray(embeddings, dtype=np.float32) if matrix.shape != (CLASS_COUNT, EMBEDDING_DIM): raise ValueError( f"{name} embeddings must have shape ({CLASS_COUNT}, {EMBEDDING_DIM}); got {matrix.shape}" ) if not np.isfinite(matrix).all(): raise ValueError(f"{name} embeddings contain non-finite values") norms = np.linalg.norm(matrix, axis=1, keepdims=True) if np.any(norms <= 1e-12) or not np.isfinite(norms).all(): raise ValueError(f"{name} embeddings contain zero-length vectors") return np.ascontiguousarray(matrix / norms, dtype=np.float32) def _load_index(path: Path, kind: str) -> tuple[np.ndarray, list[str], list[str]]: try: with np.load(path, allow_pickle=False) as archive: required = {"embeddings", "node_ids", "prompts"} missing = required.difference(archive.files) if missing: raise ValueError(f"{path.name} is missing arrays: {sorted(missing)}") embeddings = np.array(archive["embeddings"], dtype=np.float32, copy=True) ids = [str(value) for value in archive["node_ids"].tolist()] prompts = [str(value) for value in archive["prompts"].tolist()] except FileNotFoundError: raise except (OSError, ValueError) as exc: raise ValueError(f"Could not read {kind} index at {path}") from exc if len(ids) != CLASS_COUNT or len(set(ids)) != CLASS_COUNT: raise ValueError(f"{path.name} must contain {CLASS_COUNT} unique node IDs") if len(prompts) != CLASS_COUNT: raise ValueError(f"{path.name} must contain {CLASS_COUNT} prompts") return _normalize_embeddings(embeddings, kind), ids, prompts def _time_edges(duration: float, stride: float) -> np.ndarray: """Make stride-spaced edges and append the exact duration as the tail.""" full_steps = int(math.floor(duration / stride + 1e-10)) edges = np.arange(full_steps + 1, dtype=np.float64) * stride if edges.size == 0: edges = np.array([0.0], dtype=np.float64) if edges[-1] >= duration - max(1e-10, duration * 1e-12): edges[-1] = duration else: edges = np.append(edges, duration) if edges.size < 2: edges = np.array([0.0, duration], dtype=np.float64) return edges def _validate_pcm(pcm: Any) -> tuple[np.ndarray, float]: waveform = np.asarray(pcm, dtype=np.float32) if waveform.ndim != 1: raise ValueError("pcm must be a one-dimensional mono waveform") if waveform.size == 0: raise ValueError("pcm must contain at least one sample") if not np.isfinite(waveform).all(): raise ValueError("pcm contains non-finite samples") duration = waveform.size / SAMPLE_RATE if duration > 600.0: raise ValueError("Audio exceeds the 600-second limit") return np.ascontiguousarray(waveform), duration class AudioEngine: """Load CLAP2 indexes and encode synchronized audio timeline bins.""" def __init__(self, data_dir: str, device: str = "cuda", cache_dir: str | None = None): self.data_dir = Path(data_dir) self.device = str(device) self.cache_dir = cache_dir original_embeddings, original_ids, self.original_prompts = _load_index( self.data_dir / "original-index.npz", "original" ) sentence_embeddings, sentence_ids, self.sentence_prompts = _load_index( self.data_dir / "sentence-index.npz", "sentence" ) original_rows = _read_nodes(self.data_dir / "original-nodes.json") sentence_rows = _read_nodes(self.data_dir / "sentence-nodes.json") if len(set(original_ids)) != CLASS_COUNT or set(original_ids) != set(sentence_ids): raise ValueError("Original and sentence index node IDs must match exactly") # The original index defines the canonical class order. Reorder the # sentence matrix if its archive was serialized in another row order. sentence_row_by_id = {node_id: i for i, node_id in enumerate(sentence_ids)} sentence_reorder = np.asarray([sentence_row_by_id[node_id] for node_id in original_ids]) sentence_embeddings = np.ascontiguousarray(sentence_embeddings[sentence_reorder]) self.sentence_prompts = [self.sentence_prompts[i] for i in sentence_reorder] original_by_id = {str(row["id"]): row for row in original_rows} sentence_by_id = {str(row["id"]): row for row in sentence_rows} if set(original_ids) != set(original_by_id) or set(original_ids) != set(sentence_by_id): raise ValueError("Index node IDs must match their corresponding node metadata") nodes: list[dict[str, str]] = [] for node_id in original_ids: original = original_by_id[node_id] sentence_node = sentence_by_id[node_id] label = str(original.get("label", "")).strip() sentence = str(sentence_node.get("sentence", "")).strip() if not label or not sentence: raise ValueError(f"Node {node_id} is missing a label or sentence") path = _path_text(original) or _path_text(sentence_node) nodes.append({"id": node_id, "label": label, "path": path, "sentence": sentence}) self.nodes = nodes self.original_text = original_embeddings self.sentence_text = sentence_embeddings # Import model dependencies only here, so pure utilities stay usable # without downloading weights or initializing CUDA. import torch import open_clip from huggingface_hub import snapshot_download torch.set_num_threads(2) if self.device.startswith("cuda") and torch.cuda.is_available(): torch.backends.cuda.matmul.allow_tf32 = True model_dir = snapshot_download( repo_id=MODEL_REPOSITORY, revision=MODEL_REVISION, allow_patterns=["open_clip*"], cache_dir=cache_dir, ) model, preprocess_factory = open_clip.create_model_from_pretrained( f"local-dir:{model_dir}", device=self.device ) self.model = model.eval().to(self.device) self.preprocess = preprocess_factory() self._torch = torch def _amp_context(self): if self.device.startswith("cuda"): return self._torch.autocast(device_type="cuda", dtype=self._torch.bfloat16) return nullcontext() def _preprocess_windows(self, pcm: np.ndarray, starts: list[int], width: int) -> np.ndarray: torch = self._torch prepared: list[dict[str, Any]] = [] vectors: list[np.ndarray] = [] window_count = len(starts) for batch_start in range(0, window_count, 64): batch_starts = starts[batch_start : batch_start + 64] prepared.clear() for start in batch_starts: end = min(start + width, pcm.size) crop = torch.from_numpy(pcm[start:end].copy()) prepared.append(self.preprocess((crop, SAMPLE_RATE))) if not prepared: continue keys = prepared[0].keys() if any(item.keys() != prepared[0].keys() for item in prepared): raise ValueError("CLAP2 audio preprocessing returned inconsistent fields") batch = { key: torch.stack([item[key] for item in prepared]).to(self.device) for key in keys } with torch.inference_mode(), self._amp_context(): encoded = self.model.encode_audio(batch, normalize=True) encoded = encoded.float() encoded = torch.nn.functional.normalize(encoded, dim=-1) array = encoded.cpu().numpy().astype(np.float32, copy=False) if array.shape != (len(batch_starts), EMBEDDING_DIM): raise ValueError(f"CLAP2 returned unexpected audio embedding shape {array.shape}") if not np.isfinite(array).all(): raise ValueError("CLAP2 returned non-finite audio embeddings") vectors.append(array) if not vectors: raise ValueError("No audio windows were produced") return np.concatenate(vectors, axis=0) def _encode_sliding(self, pcm: np.ndarray, duration: float, window_seconds: float, stride_seconds: float): window = min(window_seconds, duration) width = max(1, int(round(window * SAMPLE_RATE))) hop = max(1, int(round(stride_seconds * SAMPLE_RATE))) if pcm.size <= width: starts = [0] else: last_start = pcm.size - width starts = list(range(0, last_start + 1, hop)) if starts[-1] != last_start: starts.append(last_start) ends = [min(start + width, pcm.size) for start in starts] window_embeddings = self._preprocess_windows(pcm, starts, width) edges = _time_edges(duration, stride_seconds) output = np.zeros((edges.size - 1, EMBEDDING_DIM), dtype=np.float64) weight_sums = np.zeros(edges.size - 1, dtype=np.float64) win_starts_s = np.asarray(starts, dtype=np.float64) / SAMPLE_RATE win_ends_s = np.asarray(ends, dtype=np.float64) / SAMPLE_RATE for bin_i, (bin_start, bin_end) in enumerate(zip(edges[:-1], edges[1:])): overlap = np.maximum( 0.0, np.minimum(win_ends_s, bin_end) - np.maximum(win_starts_s, bin_start), ) selected = overlap > 1e-12 if not selected.any(): raise RuntimeError(f"No audio windows overlap time bin {bin_i}") weights = overlap[selected] output[bin_i] = (window_embeddings[selected].astype(np.float64) * weights[:, None]).sum(axis=0) weight_sums[bin_i] = weights.sum() output /= weight_sums[:, None] if not np.isfinite(output).all(): raise ValueError("Audio embedding aggregation produced non-finite values") return output.astype(np.float32), edges, len(starts) def _encode_contextual(self, pcm: np.ndarray, duration: float, stride_seconds: float): if duration > 30.0: raise ValueError("Contextual patches support clips up to 30 seconds") torch = self._torch import torch.nn.functional as F from torch.nn.utils.rnn import pad_sequence tower = self.model.audio vit = tower.encoder.vit if getattr(vit, "pool_include_prefix", False) or vit.attn_pool.pos_embed is not None: raise RuntimeError("CLAP2 contextual pooling requires prefix exclusion and no pool position embedding") edges = _time_edges(duration, stride_seconds) waveform = torch.from_numpy(pcm.copy()) item = self.preprocess((waveform, SAMPLE_RATE)) if "patch_coord" not in item: raise RuntimeError("CLAP2 preprocessing did not provide patch_coord") coords = item["patch_coord"] if isinstance(coords, torch.Tensor): coords_np = coords.detach().cpu().numpy() else: coords_np = np.asarray(coords) if coords_np.ndim != 2 or coords_np.shape[1] < 2: raise RuntimeError("CLAP2 patch_coord has an unexpected shape") batch = { key: pad_sequence([value], batch_first=True).to(self.device) for key, value in item.items() } with torch.inference_mode(), self._amp_context(): features = vit.forward_features(**batch) patches = features["patches"] time_index = coords_np[:, 1].astype(np.float64) patch_starts = time_index * 0.16 patch_ends = np.minimum((time_index + 1.0) * 0.16, duration) real_durations = np.maximum(patch_ends - patch_starts, 1e-9) hidden = patches[0, vit.num_prefix_tokens : vit.num_prefix_tokens + len(coords_np)] selected_hidden = [] selected_priors = [] for bin_start, bin_end in zip(edges[:-1], edges[1:]): overlap = np.maximum( 0.0, np.minimum(patch_ends, bin_end) - np.maximum(patch_starts, bin_start), ) indices = np.flatnonzero(overlap > 1e-8) if indices.size == 0: raise RuntimeError( f"No CLAP2 audio patches overlap time bin [{bin_start:.3f}, {bin_end:.3f}]" ) selected_hidden.append(hidden[torch.as_tensor(indices, device=hidden.device)]) prior = np.log(overlap[indices] / real_durations[indices]) selected_priors.append( torch.as_tensor(prior, device=hidden.device, dtype=hidden.dtype) ) x = pad_sequence(selected_hidden, batch_first=True) mask = pad_sequence( selected_priors, batch_first=True, padding_value=float("-inf") )[:, None, None, :] pooled = vit.attn_pool(x, attn_mask=mask) pooled = vit.head(vit.head_drop(vit.fc_norm(pooled))) if getattr(tower, "pre_norm", False): pooled = F.normalize(pooled, dim=-1) embeddings = F.normalize(tower.proj(pooled).float(), dim=-1) result = embeddings.detach().cpu().numpy().astype(np.float32, copy=False) if result.shape != (edges.size - 1, EMBEDDING_DIM): raise ValueError(f"CLAP2 contextual pooling returned unexpected shape {result.shape}") if not np.isfinite(result).all(): raise ValueError("CLAP2 contextual pooling returned non-finite embeddings") return result, edges def encode( self, pcm: np.ndarray, window_seconds: float = 2.0, stride_seconds: float = 0.2, analysis_method: str = "Sliding windows", ) -> dict[str, Any]: waveform, duration = _validate_pcm(pcm) window = _finite_float(window_seconds, "window_seconds") stride = _finite_float(stride_seconds, "stride_seconds") if not 0.1 <= stride <= 1.0: raise ValueError("stride_seconds must be between 0.1 and 1.0") method = str(analysis_method).strip() if method not in {"Sliding windows", "Contextual patches"}: raise ValueError("analysis_method must be 'Sliding windows' or 'Contextual patches'") if method == "Sliding windows": if not 0.5 <= window <= 10.0: raise ValueError("window_seconds must be between 0.5 and 10.0") if stride > window: raise ValueError("stride_seconds cannot exceed window_seconds") embeddings, edges, window_count = self._encode_sliding(waveform, duration, window, stride) else: if duration > 30.0: raise ValueError("Contextual patches support clips up to 30 seconds") embeddings, edges = self._encode_contextual(waveform, duration, stride) window_count = 1 return { "embeddings": np.ascontiguousarray(embeddings, dtype=np.float32), "bin_edges": np.asarray(edges, dtype=np.float64), "duration_seconds": float(duration), "window_count": int(window_count), "window_seconds": float(window), "stride_seconds": float(stride), "analysis_method": method, } def smooth_scores(scores: Any, bin_edges: Any, method: str, width: float) -> np.ndarray: """Smooth per-bin class scores using duration-aware sparse weights.""" matrix = np.asarray(scores) if matrix.ndim != 2 or matrix.shape[0] == 0 or matrix.shape[1] == 0: raise ValueError("scores must be a nonempty two-dimensional matrix") if not np.isfinite(matrix).all(): raise ValueError("scores contain non-finite values") edges = np.asarray(bin_edges, dtype=np.float64) if edges.ndim != 1 or edges.size != matrix.shape[0] + 1: raise ValueError("bin_edges must have length N + 1") if not np.isfinite(edges).all() or np.any(np.diff(edges) <= 0): raise ValueError("bin_edges must be finite and strictly increasing") normalized_method = str(method).strip().lower() if normalized_method in {"none", ""}: return np.array(matrix, copy=True) if normalized_method not in {"moving average", "gaussian"}: raise ValueError("method must be None, Moving average, or Gaussian") width_seconds = _finite_float(width, "width") if width_seconds <= 0: raise ValueError("width must be positive for score smoothing") from scipy.sparse import csr_matrix starts = edges[:-1] ends = edges[1:] centers = (starts + ends) * 0.5 durations = ends - starts rows: list[int] = [] columns: list[int] = [] values: list[float] = [] if normalized_method == "gaussian": sigma = width_seconds / 2.35482 radius = 3.0 * sigma else: radius = width_seconds * 0.5 for i, center in enumerate(centers): left = center - radius right = center + radius first = max(0, int(np.searchsorted(ends, left, side="right"))) last = min(len(centers), int(np.searchsorted(starts, right, side="left"))) js = np.arange(first, max(first, last), dtype=np.int64) if normalized_method == "moving average": weights = np.maximum( 0.0, np.minimum(ends[js], right) - np.maximum(starts[js], left), ) else: weights = durations[js] * np.exp(-0.5 * ((centers[js] - center) / sigma) ** 2) keep = weights > 0 js = js[keep] weights = weights[keep] total = float(weights.sum()) if total <= 0: js = np.array([i], dtype=np.int64) weights = np.array([1.0], dtype=np.float64) total = 1.0 rows.extend([i] * len(js)) columns.extend(js.tolist()) values.extend((weights / total).tolist()) weight_matrix = csr_matrix((values, (rows, columns)), shape=(len(centers), len(centers))) result = weight_matrix @ matrix.astype(np.float64, copy=False) return np.asarray(result, dtype=matrix.dtype if np.issubdtype(matrix.dtype, np.floating) else np.float32) def make_payload( nodes: list[dict[str, Any]], scores_original: Any, scores_sentences: Any, bin_edges: Any, settings: dict[str, Any], audio_url: str, ) -> dict[str, Any]: """Build top-five retrieval rows with stable score-tie ordering.""" edge_array = np.asarray(bin_edges, dtype=np.float64) if edge_array.ndim != 1 or edge_array.size < 2 or not np.isfinite(edge_array).all(): raise ValueError("bin_edges must be a finite one-dimensional array") if np.any(np.diff(edge_array) <= 0): raise ValueError("bin_edges must be strictly increasing") count = edge_array.size - 1 if len(nodes) != CLASS_COUNT: raise ValueError(f"nodes must contain exactly {CLASS_COUNT} classes") original = np.asarray(scores_original) sentence = np.asarray(scores_sentences) expected = (count, CLASS_COUNT) if original.shape != expected or sentence.shape != expected: raise ValueError(f"score arrays must both have shape {expected}") if not np.isfinite(original).all() or not np.isfinite(sentence).all(): raise ValueError("score arrays contain non-finite values") ids = [str(node.get("id", "")) for node in nodes] if any(not node_id for node_id in ids) or len(set(ids)) != CLASS_COUNT: raise ValueError("nodes must have unique non-empty IDs") if any(not str(node.get("label", "")).strip() for node in nodes): raise ValueError("each node must have a label") if any(not str(node.get("sentence", "")).strip() for node in nodes): raise ValueError("each node must have a sentence") class_indices = np.arange(CLASS_COUNT, dtype=np.int64) top_original: list[list[dict[str, Any]]] = [] top_sentences: list[list[dict[str, Any]]] = [] for bin_i in range(count): # lexsort's last key is primary: descending score, then ascending # taxonomy index for exact ties; it never depends on hash/random state. order_original = np.lexsort((class_indices, -original[bin_i]))[:5] order_sentence = np.lexsort((class_indices, -sentence[bin_i]))[:5] top_original.append( [ { "id": ids[j], "label": str(nodes[j]["label"]), "taxonomy_path": nodes[j].get("path", ""), "score": float(original[bin_i, j]), } for j in order_original ] ) top_sentences.append( [ { "id": ids[j], "label": str(nodes[j]["label"]), "sentence": str(nodes[j]["sentence"]), "taxonomy_path": nodes[j].get("path", ""), "score": float(sentence[bin_i, j]), } for j in order_sentence ] ) return { "audio_url": str(audio_url), "duration_seconds": float(edge_array[-1] - edge_array[0]), "bin_edges": edge_array.tolist(), "original_top5": top_original, "sentences_top5": top_sentences, "settings": dict(settings), }