Add PrimeTTS v2-Stream-Clean: token-level input + v2-clean audio (RIGHT=16)

v2streamclean_streaming/README.md

@@ -0,0 +1,25 @@
+# PrimeTTS v2-Stream-Clean — token-level streaming, v2-clean audio
+
+The definitive streaming variant: **token-level band-attention encoder** (input
+streams incrementally) **+ v2's non-causal clean vocoder** (no parasite noise).
+Reconciles token-level input streaming with clean audio — the causal v2-Stream
+sacrificed quality unnecessarily; this doesn't.
+
+- `v2streamclean_enc.onnx` — text (x,tone,lang,x_lengths,noise_scale,length_scale) → z[1,192,T]. Band encoder, token-level. Run **once** per phrase.
+- `v2streamclean_dec.onnx` — z[1,192,Tc] → wav[1,1,Tc·256]. Clean non-causal vocoder. Run **per chunk**, overlap-save.
+- `onnx_stream.py` — reference runner (uses the right params).
+
+**Streaming params: chunk = 24, left = 64, RIGHT = 16** (the clean non-causal
+vocoder needs 16 future frames for bit-exact chunking; the causal one used 4).
+16 kHz, zh-TW + English.
+
+```python
+from onnx_stream import StreamingTTS   # RIGHT=16 baked in
+tts = StreamingTTS("v2streamclean_enc.onnx", "v2streamclean_dec.onnx")
+z = tts.encode(phone_ids, tone_ids, lang_ids)      # once
+for pcm in tts.stream(z): play(pcm)                # per 24-frame chunk, clean audio
+```
+Frontend (text→ids): g2pw bopomofo + g2p_en, 88 syms/6 tones/2 langs, add_blank.
+sherpa-onnx: `OfflineTtsMbistftStreamModel(enc, dec, num_threads=2, right_lookahead=16)`.
+
+License: Apache-2.0 · part of [Luigi/PrimeTTS](https://huggingface.co/Luigi/PrimeTTS).

v2streamclean_streaming/onnx_stream.py

@@ -0,0 +1,92 @@
+"""Reference streaming TTS runner over the split v2-Stream ONNX models — the exact
+orchestration a sherpa-onnx C++ runner should mirror.
+
+  enc.onnx : (x,tone,lang,x_lengths,noise_scale,length_scale) -> z[1,192,T]   (once)
+  dec.onnx : z[1,192,Tc] -> wav[1,1,Tc*256]                                    (per chunk)
+
+Streaming = run enc once, then decode z in CHUNK-frame steps via OVERLAP-SAVE:
+for chunk frames [a,b) decode z[:, :, a-LEFT : b+RIGHT] and keep the middle
+(b-a)*256 samples. Bit-exact vs the monolithic model (validated: cos 1.000000,
+maxerr ~1e-6). First audio arrives after enc + one chunk instead of the whole
+utterance.
+
+Usage:
+  python -m streaming.onnx_stream --enc <enc.onnx> --dec <dec.onnx> --ids <parity_inputs.json> [--i 0]
+  (or --text "..." with the g2pw frontend available)
+"""
+from __future__ import annotations
+import argparse, json, time
+import numpy as np
+import onnxruntime as ort
+
+C, HOP, CHUNK, LEFT, RIGHT = 192, 256, 24, 64, 16  # non-causal clean vocoder needs 16 (causal was 4)
+
+
+def _blank(seq):
+    o = [0] * (2 * len(seq) + 1)
+    o[1::2] = seq
+    return np.array([o], np.int64)
+
+
+class StreamingTTS:
+    def __init__(self, enc_path, dec_path, threads=2):
+        so = ort.SessionOptions(); so.intra_op_num_threads = threads; so.inter_op_num_threads = 1
+        self.enc = ort.InferenceSession(enc_path, so, providers=["CPUExecutionProvider"])
+        self.dec = ort.InferenceSession(dec_path, so, providers=["CPUExecutionProvider"])
+
+    def encode(self, phone_ids, tone_ids, lang_ids, noise_scale=0.667, length_scale=1.0):
+        x, tn, lg = _blank(phone_ids), _blank(tone_ids), _blank(lang_ids)
+        return self.enc.run(None, {
+            "x": x, "tone": tn, "lang": lg,
+            "x_lengths": np.array([x.shape[1]], np.int64),
+            "noise_scale": np.array([noise_scale], np.float32),
+            "length_scale": np.array([length_scale], np.float32)})[0]  # [1,192,T]
+
+    def stream(self, z):
+        """Yield audio chunks (np.float32) as z is decoded chunk-by-chunk."""
+        T = z.shape[2]
+        for a in range(0, T, CHUNK):
+            b = min(a + CHUNK, T); s0 = max(0, a - LEFT); e = min(T, b + RIGHT)
+            w = self.dec.run(None, {"z": z[:, :, s0:e]})[0].reshape(-1)
+            off = (a - s0) * HOP; keep = (b - a) * HOP
+            yield w[off:off + keep]
+
+    def synth(self, phone_ids, tone_ids, lang_ids, **kw):
+        z = self.encode(phone_ids, tone_ids, lang_ids, **kw)
+        return np.concatenate(list(self.stream(z))) if z.shape[2] else np.zeros(0, np.float32)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--enc", default="/home/luigi/mbvits_run/v2stream_split/v2stream_enc.onnx")
+    ap.add_argument("--dec", default="/home/luigi/mbvits_run/v2stream_split/v2stream_dec.onnx")
+    ap.add_argument("--ids", default="/home/luigi/mbvits_run/parity_inputs.json")
+    ap.add_argument("--i", type=int, default=0)
+    ap.add_argument("--text", default=None)
+    ap.add_argument("--out", default="/home/luigi/mbvits_run/onnx_stream_demo.wav")
+    ap.add_argument("--threads", type=int, default=2)
+    a = ap.parse_args()
+    tts = StreamingTTS(a.enc, a.dec, a.threads)
+
+    if a.text:
+        import sys; sys.path.insert(0, "/home/luigi/primetts-space")
+        import frontend_bopomofo as F
+        o = F.text_to_ids(a.text); p, t, l = o["phone_ids"], o["tone_ids"], o["lang_ids"]
+    else:
+        r = json.load(open(a.ids))["rows"][a.i]; p, t, l = r["phone_ids"], r["tone_ids"], r["lang_ids"]
+
+    t0 = time.perf_counter(); z = tts.encode(p, t, l); t_enc = time.perf_counter() - t0
+    chunks = []; tfirst = None
+    for c in tts.stream(z):
+        chunks.append(c)
+        if tfirst is None: tfirst = time.perf_counter() - t0
+    total = time.perf_counter() - t0
+    wav = np.concatenate(chunks); audio_s = len(wav) / 16000
+    pk = np.max(np.abs(wav)); wav = wav * (0.97 / pk) if pk > 1e-6 else wav
+    import soundfile as sf; sf.write(a.out, wav.astype(np.float32), 16000)
+    print(f"frames={z.shape[2]} audio={audio_s:.2f}s  enc={t_enc*1e3:.0f}ms "
+          f"first-audio={tfirst*1e3:.0f}ms  total={total*1e3:.0f}ms RTF={total/audio_s:.3f} -> {a.out}")
+
+
+if __name__ == "__main__":
+    main()

v2streamclean_streaming/v2streamclean_dec.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfed5c287d651d46d48b63cec76b13b27732b2344069af77f02c458abf4801fc
+size 54886617

v2streamclean_streaming/v2streamclean_enc.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddf7a0e2e2042899992ff669401b3ee2dce2474c763a47243ae78119f9effe20
+size 55538543