KV1-Demo / app.py
Banaxi-Tech's picture
Show explicit Send and Stop buttons on chat textbox
1c99296 verified
Raw History Blame Contribute Delete
7.79 kB
"""KV1 chat demo for a Gradio Hugging Face ZeroGPU Space."""
import logging
import os
from threading import Event, Thread
# Import spaces before torch so ZeroGPU can install its CUDA hooks.
import spaces
import gradio as gr
import torch
from transformers import (
AutoModelForCausalLM,
AutoTokenizer,
StoppingCriteria,
StoppingCriteriaList,
TextIteratorStreamer,
)
MODEL_ID = "klein-ml/KV1-1.2B-Prerelease"
MODEL_REVISION = os.getenv("MODEL_REVISION", "main")
LOGGER = logging.getLogger(__name__)
class StopOnEvent(StoppingCriteria):
def __init__(self, event):
self.event = event
def __call__(self, input_ids, scores, **kwargs):
return self.event.is_set()
class ChatModel:
def __init__(self, tokenizer, model):
self.tokenizer = tokenizer
self.model = model
# Ignore the very large sentinel used by tokenizers without a limit.
limits = [
value
for value in (
getattr(model.config, "max_position_embeddings", None),
getattr(tokenizer, "model_max_length", None),
)
if isinstance(value, int) and 0 < value < 1_000_000
]
self.context_limit = min(limits) if limits else 4096
def stream(self, message, history, system_prompt, max_new_tokens, temperature, top_p):
if not message.strip():
raise gr.Error("Enter a message first.")
messages = []
if system_prompt.strip():
messages.append({"role": "system", "content": system_prompt.strip()})
messages.extend(
{"role": item["role"], "content": item["content"]}
for item in history
if item["role"] in ("user", "assistant")
and isinstance(item["content"], str)
)
messages.append({"role": "user", "content": message})
try:
prompt = self.tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
except (ValueError, TypeError, KeyError) as exc:
raise gr.Error(
"Could not format this conversation with the model's chat template. "
"Try an empty system prompt. If that fails, check the tokenizer's "
"chat_template in the model repository."
) from exc
# The template already supplies any required special tokens.
inputs = self.tokenizer(prompt, return_tensors="pt", add_special_tokens=False)
max_new_tokens = int(max_new_tokens)
if inputs.input_ids.shape[-1] + max_new_tokens > self.context_limit:
raise gr.Error(
f"This conversation plus the requested reply exceeds the "
f"{self.context_limit:,}-token context. Clear the chat, shorten your "
"message, or reduce Max new tokens."
)
inputs = inputs.to(self.model.device)
streamer = TextIteratorStreamer(
self.tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=110
)
cancelled = Event()
errors = []
options = dict(
**inputs,
streamer=streamer,
max_new_tokens=max_new_tokens,
max_time=100,
do_sample=temperature > 0,
stopping_criteria=StoppingCriteriaList([StopOnEvent(cancelled)]),
)
pad_token_id = self.tokenizer.pad_token_id
if pad_token_id is None:
pad_token_id = self.tokenizer.eos_token_id
if pad_token_id is not None:
options["pad_token_id"] = pad_token_id
if temperature > 0:
options.update(temperature=float(temperature), top_p=float(top_p))
def generate():
try:
# inference_mode is thread-local, so enter it in the worker.
with torch.inference_mode():
self.model.generate(**options)
except Exception as exc:
errors.append(exc)
streamer.end() # Wake the consumer even if generate() fails.
worker = Thread(target=generate, daemon=True)
worker.start()
try:
response = ""
for chunk in streamer:
response += chunk
yield response
if errors:
raise errors[0]
except GeneratorExit:
raise
except Exception as exc:
LOGGER.exception("Generation failed")
raise gr.Error("Generation failed. Check the Space logs for details.") from exc
finally:
cancelled.set()
# Do not leave generation using the GPU after the request releases it.
worker.join()
def load_model():
token = os.getenv("HF_TOKEN")
if not token:
raise RuntimeError(
"Add HF_TOKEN as a Space secret with read access to " + MODEL_ID + "."
)
common = dict(token=token, revision=MODEL_REVISION, trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, **common)
device = "cuda" if torch.cuda.is_available() else "cpu"
model = AutoModelForCausalLM.from_pretrained(
MODEL_ID,
**common,
dtype=torch.bfloat16 if device == "cuda" else torch.float32,
).to(device).eval()
return ChatModel(tokenizer, model)
def create_demo(chat_model):
@spaces.GPU(duration=120)
def respond(message, history, system_prompt, max_new_tokens, temperature, top_p):
yield from chat_model.stream(
message, history, system_prompt, max_new_tokens, temperature, top_p
)
with gr.Blocks(title="KV1 · Chat", theme=gr.themes.Soft()) as demo:
gr.Markdown(
"# KV1 · Chat\n"
"Try [KV1 1.2B Prerelease](https://huggingface.co/"
"klein-ml/KV1-1.2B-Prerelease). Responses stream with Markdown, "
"tables, and syntax-highlighted code."
)
with gr.Accordion("Generation settings", open=False):
system_prompt = gr.Textbox(
label="System prompt", value="", placeholder="Optional instructions…",
lines=2,
)
with gr.Row():
max_new_tokens = gr.Slider(32, 2048, value=512, step=32, label="Max new tokens")
temperature = gr.Slider(0, 2, value=0.7, step=0.05, label="Temperature (0 = greedy)")
top_p = gr.Slider(0.05, 1, value=0.9, step=0.05, label="Top-p")
gr.ChatInterface(
fn=respond,
type="messages",
chatbot=gr.Chatbot(
type="messages", label="KV1", height=540,
render_markdown=True, sanitize_html=True,
show_copy_button=True,
),
textbox=gr.Textbox(
placeholder="Message KV1…", lines=2, max_lines=8,
submit_btn="Send", stop_btn="Stop",
),
additional_inputs=[system_prompt, max_new_tokens, temperature, top_p],
examples=[
["Explain how a rainbow forms.", "", 512, 0.7, 0.9],
["Write a Python function that checks whether a word is a palindrome.", "", 512, 0.7, 0.9],
["Compare lists and tuples in Python using a Markdown table.", "", 512, 0.7, 0.9],
],
cache_examples=False,
concurrency_limit=1,
save_history=False,
)
gr.Markdown("Prerelease model · Answers may be incorrect. · Powered by ZeroGPU")
return demo.queue(max_size=20, default_concurrency_limit=1)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
# Load and place the model at startup, outside the GPU-decorated callback.
create_demo(load_model()).launch()