Spaces:
Paused
Paused
Download engine.py from TTS-AGI/clap2-live-taxonomy: direct link, hf CLI and curl.
- Browser
- Download file 27 kB
-
https://huggingface.co/spaces/TTS-AGI/clap2-live-taxonomy/resolve/main/engine.py
- Command line
-
hf download hf://spaces/TTS-AGI/clap2-live-taxonomy/engine.py
-
curl -L -o engine.py https://huggingface.co/spaces/TTS-AGI/clap2-live-taxonomy/resolve/main/engine.py
27 kB
| """CLAP2-backed audio timeline engine for the public taxonomy demo.""" | |
| from __future__ import annotations | |
| import bisect | |
| import json | |
| import math | |
| import os | |
| import subprocess | |
| import tempfile | |
| from contextlib import nullcontext | |
| from pathlib import Path | |
| from typing import Any | |
| import numpy as np | |
| SAMPLE_RATE = 32_000 | |
| EMBEDDING_DIM = 640 | |
| CLASS_COUNT = 4_416 | |
| MODEL_REPOSITORY = "laion/clap2-medium-audioxl-s1B" | |
| MODEL_REVISION = "8e299e83c08a291711490105e82a345abadb5cdb" | |
| def _finite_float(value: Any, name: str) -> float: | |
| try: | |
| result = float(value) | |
| except (TypeError, ValueError) as exc: | |
| raise ValueError(f"{name} must be a finite number") from exc | |
| if not math.isfinite(result): | |
| raise ValueError(f"{name} must be a finite number") | |
| return result | |
| def load_audio(path: str, max_seconds: float = 600) -> tuple[np.ndarray, int]: | |
| """Decode a complete mono waveform at 32 kHz with ffmpeg. | |
| Accepted uploads are decoded through EOF, so duration does not depend on | |
| container duration estimates (which can truncate VBR MP3 files). For an | |
| overlong input, decoding stops as soon as more than ``max_seconds`` worth | |
| of samples arrives, then rejects the file explicitly. | |
| """ | |
| source = Path(path) | |
| if not source.is_file(): | |
| raise FileNotFoundError(f"Audio file does not exist: {source}") | |
| limit = _finite_float(max_seconds, "max_seconds") | |
| if limit <= 0: | |
| raise ValueError("max_seconds must be greater than zero") | |
| if limit > 600.0: | |
| raise ValueError("max_seconds cannot exceed the 600-second upload limit") | |
| max_samples = int(math.floor(limit * SAMPLE_RATE + 1e-9)) | |
| command = [ | |
| "ffmpeg", | |
| "-hide_banner", | |
| "-loglevel", | |
| "error", | |
| "-nostdin", | |
| "-protocol_whitelist", | |
| "file,pipe", | |
| "-i", | |
| str(source), | |
| "-map", | |
| "0:a:0", | |
| "-vn", | |
| "-sn", | |
| "-dn", | |
| "-ac", | |
| "1", | |
| "-ar", | |
| str(SAMPLE_RATE), | |
| "-acodec", | |
| "pcm_f32le", | |
| "-f", | |
| "f32le", | |
| "pipe:1", | |
| ] | |
| stderr_file = tempfile.TemporaryFile() | |
| process: subprocess.Popen[bytes] | None = None | |
| chunks: list[bytes] = [] | |
| total_bytes = 0 | |
| over_limit = False | |
| try: | |
| process = subprocess.Popen( | |
| command, | |
| stdout=subprocess.PIPE, | |
| stderr=stderr_file, | |
| bufsize=0, | |
| ) | |
| assert process.stdout is not None | |
| while True: | |
| block = process.stdout.read(65_536) | |
| if not block: | |
| break | |
| chunks.append(block) | |
| total_bytes += len(block) | |
| # Reads may end within a sample; use floor so a partial trailing | |
| # sample cannot cause a false over-limit rejection. | |
| if total_bytes // 4 > max_samples: | |
| over_limit = True | |
| process.kill() | |
| break | |
| process.stdout.close() | |
| return_code = process.wait() | |
| if over_limit: | |
| raise ValueError(f"Audio exceeds the {limit:g}-second limit") | |
| if return_code != 0: | |
| stderr_file.seek(0) | |
| diagnostic = stderr_file.read().decode("utf-8", errors="replace").strip() | |
| raise RuntimeError(f"ffmpeg could not decode audio: {diagnostic or 'unknown error'}") | |
| if total_bytes % 4: | |
| raise RuntimeError("ffmpeg returned an incomplete float32 audio sample") | |
| pcm = np.frombuffer(b"".join(chunks), dtype="<f4").astype(np.float32, copy=True) | |
| if pcm.size == 0: | |
| raise ValueError("Audio contains no decodable samples") | |
| if not np.isfinite(pcm).all(): | |
| raise ValueError("Decoded audio contains non-finite samples") | |
| if pcm.size > max_samples: | |
| raise ValueError(f"Audio exceeds the {limit:g}-second limit") | |
| return pcm, SAMPLE_RATE | |
| except BaseException: | |
| if process is not None and process.poll() is None: | |
| if process.stdout is not None: | |
| try: | |
| process.stdout.close() | |
| except OSError: | |
| pass | |
| try: | |
| process.kill() | |
| except OSError: | |
| pass | |
| process.wait() | |
| raise | |
| finally: | |
| stderr_file.close() | |
| def _read_nodes(path: Path) -> list[dict[str, Any]]: | |
| try: | |
| payload = json.loads(path.read_text(encoding="utf-8")) | |
| except FileNotFoundError: | |
| raise | |
| except (json.JSONDecodeError, OSError) as exc: | |
| raise ValueError(f"Could not read node metadata from {path}") from exc | |
| rows = payload.get("nodes") if isinstance(payload, dict) else payload | |
| if not isinstance(rows, list) or len(rows) != CLASS_COUNT: | |
| raise ValueError(f"{path.name} must contain exactly {CLASS_COUNT} nodes") | |
| if any(not isinstance(row, dict) for row in rows): | |
| raise ValueError(f"{path.name} contains a non-object node") | |
| ids = [str(row.get("id", "")) for row in rows] | |
| if any(not node_id for node_id in ids) or len(set(ids)) != CLASS_COUNT: | |
| raise ValueError(f"{path.name} contains missing or duplicate node IDs") | |
| return rows | |
| def _path_text(node: dict[str, Any]) -> str: | |
| value = node.get("path_text") | |
| if isinstance(value, str) and value.strip(): | |
| return value.strip() | |
| value = node.get("path", "") | |
| if isinstance(value, (list, tuple)): | |
| return " › ".join(str(part).strip() for part in value if str(part).strip()) | |
| return str(value or "").strip() | |
| def _normalize_embeddings(embeddings: np.ndarray, name: str) -> np.ndarray: | |
| matrix = np.asarray(embeddings, dtype=np.float32) | |
| if matrix.shape != (CLASS_COUNT, EMBEDDING_DIM): | |
| raise ValueError( | |
| f"{name} embeddings must have shape ({CLASS_COUNT}, {EMBEDDING_DIM}); got {matrix.shape}" | |
| ) | |
| if not np.isfinite(matrix).all(): | |
| raise ValueError(f"{name} embeddings contain non-finite values") | |
| norms = np.linalg.norm(matrix, axis=1, keepdims=True) | |
| if np.any(norms <= 1e-12) or not np.isfinite(norms).all(): | |
| raise ValueError(f"{name} embeddings contain zero-length vectors") | |
| return np.ascontiguousarray(matrix / norms, dtype=np.float32) | |
| def _load_index(path: Path, kind: str) -> tuple[np.ndarray, list[str], list[str]]: | |
| try: | |
| with np.load(path, allow_pickle=False) as archive: | |
| required = {"embeddings", "node_ids", "prompts"} | |
| missing = required.difference(archive.files) | |
| if missing: | |
| raise ValueError(f"{path.name} is missing arrays: {sorted(missing)}") | |
| embeddings = np.array(archive["embeddings"], dtype=np.float32, copy=True) | |
| ids = [str(value) for value in archive["node_ids"].tolist()] | |
| prompts = [str(value) for value in archive["prompts"].tolist()] | |
| except FileNotFoundError: | |
| raise | |
| except (OSError, ValueError) as exc: | |
| raise ValueError(f"Could not read {kind} index at {path}") from exc | |
| if len(ids) != CLASS_COUNT or len(set(ids)) != CLASS_COUNT: | |
| raise ValueError(f"{path.name} must contain {CLASS_COUNT} unique node IDs") | |
| if len(prompts) != CLASS_COUNT: | |
| raise ValueError(f"{path.name} must contain {CLASS_COUNT} prompts") | |
| return _normalize_embeddings(embeddings, kind), ids, prompts | |
| def _time_edges(duration: float, stride: float) -> np.ndarray: | |
| """Make stride-spaced edges and append the exact duration as the tail.""" | |
| full_steps = int(math.floor(duration / stride + 1e-10)) | |
| edges = np.arange(full_steps + 1, dtype=np.float64) * stride | |
| if edges.size == 0: | |
| edges = np.array([0.0], dtype=np.float64) | |
| if edges[-1] >= duration - max(1e-10, duration * 1e-12): | |
| edges[-1] = duration | |
| else: | |
| edges = np.append(edges, duration) | |
| if edges.size < 2: | |
| edges = np.array([0.0, duration], dtype=np.float64) | |
| return edges | |
| def _validate_pcm(pcm: Any) -> tuple[np.ndarray, float]: | |
| waveform = np.asarray(pcm, dtype=np.float32) | |
| if waveform.ndim != 1: | |
| raise ValueError("pcm must be a one-dimensional mono waveform") | |
| if waveform.size == 0: | |
| raise ValueError("pcm must contain at least one sample") | |
| if not np.isfinite(waveform).all(): | |
| raise ValueError("pcm contains non-finite samples") | |
| duration = waveform.size / SAMPLE_RATE | |
| if duration > 600.0: | |
| raise ValueError("Audio exceeds the 600-second limit") | |
| return np.ascontiguousarray(waveform), duration | |
| class AudioEngine: | |
| """Load CLAP2 indexes and encode synchronized audio timeline bins.""" | |
| def __init__(self, data_dir: str, device: str = "cuda", cache_dir: str | None = None): | |
| self.data_dir = Path(data_dir) | |
| self.device = str(device) | |
| self.cache_dir = cache_dir | |
| original_embeddings, original_ids, self.original_prompts = _load_index( | |
| self.data_dir / "original-index.npz", "original" | |
| ) | |
| sentence_embeddings, sentence_ids, self.sentence_prompts = _load_index( | |
| self.data_dir / "sentence-index.npz", "sentence" | |
| ) | |
| original_rows = _read_nodes(self.data_dir / "original-nodes.json") | |
| sentence_rows = _read_nodes(self.data_dir / "sentence-nodes.json") | |
| if len(set(original_ids)) != CLASS_COUNT or set(original_ids) != set(sentence_ids): | |
| raise ValueError("Original and sentence index node IDs must match exactly") | |
| # The original index defines the canonical class order. Reorder the | |
| # sentence matrix if its archive was serialized in another row order. | |
| sentence_row_by_id = {node_id: i for i, node_id in enumerate(sentence_ids)} | |
| sentence_reorder = np.asarray([sentence_row_by_id[node_id] for node_id in original_ids]) | |
| sentence_embeddings = np.ascontiguousarray(sentence_embeddings[sentence_reorder]) | |
| self.sentence_prompts = [self.sentence_prompts[i] for i in sentence_reorder] | |
| original_by_id = {str(row["id"]): row for row in original_rows} | |
| sentence_by_id = {str(row["id"]): row for row in sentence_rows} | |
| if set(original_ids) != set(original_by_id) or set(original_ids) != set(sentence_by_id): | |
| raise ValueError("Index node IDs must match their corresponding node metadata") | |
| nodes: list[dict[str, str]] = [] | |
| for node_id in original_ids: | |
| original = original_by_id[node_id] | |
| sentence_node = sentence_by_id[node_id] | |
| label = str(original.get("label", "")).strip() | |
| sentence = str(sentence_node.get("sentence", "")).strip() | |
| if not label or not sentence: | |
| raise ValueError(f"Node {node_id} is missing a label or sentence") | |
| path = _path_text(original) or _path_text(sentence_node) | |
| nodes.append({"id": node_id, "label": label, "path": path, "sentence": sentence}) | |
| self.nodes = nodes | |
| self.original_text = original_embeddings | |
| self.sentence_text = sentence_embeddings | |
| # Import model dependencies only here, so pure utilities stay usable | |
| # without downloading weights or initializing CUDA. | |
| import torch | |
| import open_clip | |
| from huggingface_hub import snapshot_download | |
| torch.set_num_threads(2) | |
| if self.device.startswith("cuda") and torch.cuda.is_available(): | |
| torch.backends.cuda.matmul.allow_tf32 = True | |
| model_dir = snapshot_download( | |
| repo_id=MODEL_REPOSITORY, | |
| revision=MODEL_REVISION, | |
| allow_patterns=["open_clip*"], | |
| cache_dir=cache_dir, | |
| ) | |
| model, preprocess_factory = open_clip.create_model_from_pretrained( | |
| f"local-dir:{model_dir}", device=self.device | |
| ) | |
| self.model = model.eval().to(self.device) | |
| self.preprocess = preprocess_factory() | |
| self._torch = torch | |
| def _amp_context(self): | |
| if self.device.startswith("cuda"): | |
| return self._torch.autocast(device_type="cuda", dtype=self._torch.bfloat16) | |
| return nullcontext() | |
| def _preprocess_windows(self, pcm: np.ndarray, starts: list[int], width: int) -> np.ndarray: | |
| torch = self._torch | |
| prepared: list[dict[str, Any]] = [] | |
| vectors: list[np.ndarray] = [] | |
| window_count = len(starts) | |
| for batch_start in range(0, window_count, 64): | |
| batch_starts = starts[batch_start : batch_start + 64] | |
| prepared.clear() | |
| for start in batch_starts: | |
| end = min(start + width, pcm.size) | |
| crop = torch.from_numpy(pcm[start:end].copy()) | |
| prepared.append(self.preprocess((crop, SAMPLE_RATE))) | |
| if not prepared: | |
| continue | |
| keys = prepared[0].keys() | |
| if any(item.keys() != prepared[0].keys() for item in prepared): | |
| raise ValueError("CLAP2 audio preprocessing returned inconsistent fields") | |
| batch = { | |
| key: torch.stack([item[key] for item in prepared]).to(self.device) | |
| for key in keys | |
| } | |
| with torch.inference_mode(), self._amp_context(): | |
| encoded = self.model.encode_audio(batch, normalize=True) | |
| encoded = encoded.float() | |
| encoded = torch.nn.functional.normalize(encoded, dim=-1) | |
| array = encoded.cpu().numpy().astype(np.float32, copy=False) | |
| if array.shape != (len(batch_starts), EMBEDDING_DIM): | |
| raise ValueError(f"CLAP2 returned unexpected audio embedding shape {array.shape}") | |
| if not np.isfinite(array).all(): | |
| raise ValueError("CLAP2 returned non-finite audio embeddings") | |
| vectors.append(array) | |
| if not vectors: | |
| raise ValueError("No audio windows were produced") | |
| return np.concatenate(vectors, axis=0) | |
| def _encode_sliding(self, pcm: np.ndarray, duration: float, window_seconds: float, stride_seconds: float): | |
| window = min(window_seconds, duration) | |
| width = max(1, int(round(window * SAMPLE_RATE))) | |
| hop = max(1, int(round(stride_seconds * SAMPLE_RATE))) | |
| if pcm.size <= width: | |
| starts = [0] | |
| else: | |
| last_start = pcm.size - width | |
| starts = list(range(0, last_start + 1, hop)) | |
| if starts[-1] != last_start: | |
| starts.append(last_start) | |
| ends = [min(start + width, pcm.size) for start in starts] | |
| window_embeddings = self._preprocess_windows(pcm, starts, width) | |
| edges = _time_edges(duration, stride_seconds) | |
| output = np.zeros((edges.size - 1, EMBEDDING_DIM), dtype=np.float64) | |
| weight_sums = np.zeros(edges.size - 1, dtype=np.float64) | |
| win_starts_s = np.asarray(starts, dtype=np.float64) / SAMPLE_RATE | |
| win_ends_s = np.asarray(ends, dtype=np.float64) / SAMPLE_RATE | |
| for bin_i, (bin_start, bin_end) in enumerate(zip(edges[:-1], edges[1:])): | |
| overlap = np.maximum( | |
| 0.0, | |
| np.minimum(win_ends_s, bin_end) - np.maximum(win_starts_s, bin_start), | |
| ) | |
| selected = overlap > 1e-12 | |
| if not selected.any(): | |
| raise RuntimeError(f"No audio windows overlap time bin {bin_i}") | |
| weights = overlap[selected] | |
| output[bin_i] = (window_embeddings[selected].astype(np.float64) * weights[:, None]).sum(axis=0) | |
| weight_sums[bin_i] = weights.sum() | |
| output /= weight_sums[:, None] | |
| if not np.isfinite(output).all(): | |
| raise ValueError("Audio embedding aggregation produced non-finite values") | |
| return output.astype(np.float32), edges, len(starts) | |
| def _encode_contextual(self, pcm: np.ndarray, duration: float, stride_seconds: float): | |
| if duration > 30.0: | |
| raise ValueError("Contextual patches support clips up to 30 seconds") | |
| torch = self._torch | |
| import torch.nn.functional as F | |
| from torch.nn.utils.rnn import pad_sequence | |
| tower = self.model.audio | |
| vit = tower.encoder.vit | |
| if getattr(vit, "pool_include_prefix", False) or vit.attn_pool.pos_embed is not None: | |
| raise RuntimeError("CLAP2 contextual pooling requires prefix exclusion and no pool position embedding") | |
| edges = _time_edges(duration, stride_seconds) | |
| waveform = torch.from_numpy(pcm.copy()) | |
| item = self.preprocess((waveform, SAMPLE_RATE)) | |
| if "patch_coord" not in item: | |
| raise RuntimeError("CLAP2 preprocessing did not provide patch_coord") | |
| coords = item["patch_coord"] | |
| if isinstance(coords, torch.Tensor): | |
| coords_np = coords.detach().cpu().numpy() | |
| else: | |
| coords_np = np.asarray(coords) | |
| if coords_np.ndim != 2 or coords_np.shape[1] < 2: | |
| raise RuntimeError("CLAP2 patch_coord has an unexpected shape") | |
| batch = { | |
| key: pad_sequence([value], batch_first=True).to(self.device) | |
| for key, value in item.items() | |
| } | |
| with torch.inference_mode(), self._amp_context(): | |
| features = vit.forward_features(**batch) | |
| patches = features["patches"] | |
| time_index = coords_np[:, 1].astype(np.float64) | |
| patch_starts = time_index * 0.16 | |
| patch_ends = np.minimum((time_index + 1.0) * 0.16, duration) | |
| real_durations = np.maximum(patch_ends - patch_starts, 1e-9) | |
| hidden = patches[0, vit.num_prefix_tokens : vit.num_prefix_tokens + len(coords_np)] | |
| selected_hidden = [] | |
| selected_priors = [] | |
| for bin_start, bin_end in zip(edges[:-1], edges[1:]): | |
| overlap = np.maximum( | |
| 0.0, | |
| np.minimum(patch_ends, bin_end) - np.maximum(patch_starts, bin_start), | |
| ) | |
| indices = np.flatnonzero(overlap > 1e-8) | |
| if indices.size == 0: | |
| raise RuntimeError( | |
| f"No CLAP2 audio patches overlap time bin [{bin_start:.3f}, {bin_end:.3f}]" | |
| ) | |
| selected_hidden.append(hidden[torch.as_tensor(indices, device=hidden.device)]) | |
| prior = np.log(overlap[indices] / real_durations[indices]) | |
| selected_priors.append( | |
| torch.as_tensor(prior, device=hidden.device, dtype=hidden.dtype) | |
| ) | |
| x = pad_sequence(selected_hidden, batch_first=True) | |
| mask = pad_sequence( | |
| selected_priors, batch_first=True, padding_value=float("-inf") | |
| )[:, None, None, :] | |
| pooled = vit.attn_pool(x, attn_mask=mask) | |
| pooled = vit.head(vit.head_drop(vit.fc_norm(pooled))) | |
| if getattr(tower, "pre_norm", False): | |
| pooled = F.normalize(pooled, dim=-1) | |
| embeddings = F.normalize(tower.proj(pooled).float(), dim=-1) | |
| result = embeddings.detach().cpu().numpy().astype(np.float32, copy=False) | |
| if result.shape != (edges.size - 1, EMBEDDING_DIM): | |
| raise ValueError(f"CLAP2 contextual pooling returned unexpected shape {result.shape}") | |
| if not np.isfinite(result).all(): | |
| raise ValueError("CLAP2 contextual pooling returned non-finite embeddings") | |
| return result, edges | |
| def encode( | |
| self, | |
| pcm: np.ndarray, | |
| window_seconds: float = 2.0, | |
| stride_seconds: float = 0.2, | |
| analysis_method: str = "Sliding windows", | |
| ) -> dict[str, Any]: | |
| waveform, duration = _validate_pcm(pcm) | |
| window = _finite_float(window_seconds, "window_seconds") | |
| stride = _finite_float(stride_seconds, "stride_seconds") | |
| if not 0.1 <= stride <= 1.0: | |
| raise ValueError("stride_seconds must be between 0.1 and 1.0") | |
| method = str(analysis_method).strip() | |
| if method not in {"Sliding windows", "Contextual patches"}: | |
| raise ValueError("analysis_method must be 'Sliding windows' or 'Contextual patches'") | |
| if method == "Sliding windows": | |
| if not 0.5 <= window <= 10.0: | |
| raise ValueError("window_seconds must be between 0.5 and 10.0") | |
| if stride > window: | |
| raise ValueError("stride_seconds cannot exceed window_seconds") | |
| embeddings, edges, window_count = self._encode_sliding(waveform, duration, window, stride) | |
| else: | |
| if duration > 30.0: | |
| raise ValueError("Contextual patches support clips up to 30 seconds") | |
| embeddings, edges = self._encode_contextual(waveform, duration, stride) | |
| window_count = 1 | |
| return { | |
| "embeddings": np.ascontiguousarray(embeddings, dtype=np.float32), | |
| "bin_edges": np.asarray(edges, dtype=np.float64), | |
| "duration_seconds": float(duration), | |
| "window_count": int(window_count), | |
| "window_seconds": float(window), | |
| "stride_seconds": float(stride), | |
| "analysis_method": method, | |
| } | |
| def smooth_scores(scores: Any, bin_edges: Any, method: str, width: float) -> np.ndarray: | |
| """Smooth per-bin class scores using duration-aware sparse weights.""" | |
| matrix = np.asarray(scores) | |
| if matrix.ndim != 2 or matrix.shape[0] == 0 or matrix.shape[1] == 0: | |
| raise ValueError("scores must be a nonempty two-dimensional matrix") | |
| if not np.isfinite(matrix).all(): | |
| raise ValueError("scores contain non-finite values") | |
| edges = np.asarray(bin_edges, dtype=np.float64) | |
| if edges.ndim != 1 or edges.size != matrix.shape[0] + 1: | |
| raise ValueError("bin_edges must have length N + 1") | |
| if not np.isfinite(edges).all() or np.any(np.diff(edges) <= 0): | |
| raise ValueError("bin_edges must be finite and strictly increasing") | |
| normalized_method = str(method).strip().lower() | |
| if normalized_method in {"none", ""}: | |
| return np.array(matrix, copy=True) | |
| if normalized_method not in {"moving average", "gaussian"}: | |
| raise ValueError("method must be None, Moving average, or Gaussian") | |
| width_seconds = _finite_float(width, "width") | |
| if width_seconds <= 0: | |
| raise ValueError("width must be positive for score smoothing") | |
| from scipy.sparse import csr_matrix | |
| starts = edges[:-1] | |
| ends = edges[1:] | |
| centers = (starts + ends) * 0.5 | |
| durations = ends - starts | |
| rows: list[int] = [] | |
| columns: list[int] = [] | |
| values: list[float] = [] | |
| if normalized_method == "gaussian": | |
| sigma = width_seconds / 2.35482 | |
| radius = 3.0 * sigma | |
| else: | |
| radius = width_seconds * 0.5 | |
| for i, center in enumerate(centers): | |
| left = center - radius | |
| right = center + radius | |
| first = max(0, int(np.searchsorted(ends, left, side="right"))) | |
| last = min(len(centers), int(np.searchsorted(starts, right, side="left"))) | |
| js = np.arange(first, max(first, last), dtype=np.int64) | |
| if normalized_method == "moving average": | |
| weights = np.maximum( | |
| 0.0, | |
| np.minimum(ends[js], right) - np.maximum(starts[js], left), | |
| ) | |
| else: | |
| weights = durations[js] * np.exp(-0.5 * ((centers[js] - center) / sigma) ** 2) | |
| keep = weights > 0 | |
| js = js[keep] | |
| weights = weights[keep] | |
| total = float(weights.sum()) | |
| if total <= 0: | |
| js = np.array([i], dtype=np.int64) | |
| weights = np.array([1.0], dtype=np.float64) | |
| total = 1.0 | |
| rows.extend([i] * len(js)) | |
| columns.extend(js.tolist()) | |
| values.extend((weights / total).tolist()) | |
| weight_matrix = csr_matrix((values, (rows, columns)), shape=(len(centers), len(centers))) | |
| result = weight_matrix @ matrix.astype(np.float64, copy=False) | |
| return np.asarray(result, dtype=matrix.dtype if np.issubdtype(matrix.dtype, np.floating) else np.float32) | |
| def make_payload( | |
| nodes: list[dict[str, Any]], | |
| scores_original: Any, | |
| scores_sentences: Any, | |
| bin_edges: Any, | |
| settings: dict[str, Any], | |
| audio_url: str, | |
| ) -> dict[str, Any]: | |
| """Build top-five retrieval rows with stable score-tie ordering.""" | |
| edge_array = np.asarray(bin_edges, dtype=np.float64) | |
| if edge_array.ndim != 1 or edge_array.size < 2 or not np.isfinite(edge_array).all(): | |
| raise ValueError("bin_edges must be a finite one-dimensional array") | |
| if np.any(np.diff(edge_array) <= 0): | |
| raise ValueError("bin_edges must be strictly increasing") | |
| count = edge_array.size - 1 | |
| if len(nodes) != CLASS_COUNT: | |
| raise ValueError(f"nodes must contain exactly {CLASS_COUNT} classes") | |
| original = np.asarray(scores_original) | |
| sentence = np.asarray(scores_sentences) | |
| expected = (count, CLASS_COUNT) | |
| if original.shape != expected or sentence.shape != expected: | |
| raise ValueError(f"score arrays must both have shape {expected}") | |
| if not np.isfinite(original).all() or not np.isfinite(sentence).all(): | |
| raise ValueError("score arrays contain non-finite values") | |
| ids = [str(node.get("id", "")) for node in nodes] | |
| if any(not node_id for node_id in ids) or len(set(ids)) != CLASS_COUNT: | |
| raise ValueError("nodes must have unique non-empty IDs") | |
| if any(not str(node.get("label", "")).strip() for node in nodes): | |
| raise ValueError("each node must have a label") | |
| if any(not str(node.get("sentence", "")).strip() for node in nodes): | |
| raise ValueError("each node must have a sentence") | |
| class_indices = np.arange(CLASS_COUNT, dtype=np.int64) | |
| top_original: list[list[dict[str, Any]]] = [] | |
| top_sentences: list[list[dict[str, Any]]] = [] | |
| for bin_i in range(count): | |
| # lexsort's last key is primary: descending score, then ascending | |
| # taxonomy index for exact ties; it never depends on hash/random state. | |
| order_original = np.lexsort((class_indices, -original[bin_i]))[:5] | |
| order_sentence = np.lexsort((class_indices, -sentence[bin_i]))[:5] | |
| top_original.append( | |
| [ | |
| { | |
| "id": ids[j], | |
| "label": str(nodes[j]["label"]), | |
| "taxonomy_path": nodes[j].get("path", ""), | |
| "score": float(original[bin_i, j]), | |
| } | |
| for j in order_original | |
| ] | |
| ) | |
| top_sentences.append( | |
| [ | |
| { | |
| "id": ids[j], | |
| "label": str(nodes[j]["label"]), | |
| "sentence": str(nodes[j]["sentence"]), | |
| "taxonomy_path": nodes[j].get("path", ""), | |
| "score": float(sentence[bin_i, j]), | |
| } | |
| for j in order_sentence | |
| ] | |
| ) | |
| return { | |
| "audio_url": str(audio_url), | |
| "duration_seconds": float(edge_array[-1] - edge_array[0]), | |
| "bin_edges": edge_array.tolist(), | |
| "original_top5": top_original, | |
| "sentences_top5": top_sentences, | |
| "settings": dict(settings), | |
| } | |