"""KV1 chat demo for a Gradio Hugging Face ZeroGPU Space.""" import logging import os from threading import Event, Thread # Import spaces before torch so ZeroGPU can install its CUDA hooks. import spaces import gradio as gr import torch from transformers import ( AutoModelForCausalLM, AutoTokenizer, StoppingCriteria, StoppingCriteriaList, TextIteratorStreamer, ) MODEL_ID = "klein-ml/KV1-1.2B-Prerelease" MODEL_REVISION = os.getenv("MODEL_REVISION", "main") LOGGER = logging.getLogger(__name__) class StopOnEvent(StoppingCriteria): def __init__(self, event): self.event = event def __call__(self, input_ids, scores, **kwargs): return self.event.is_set() class ChatModel: def __init__(self, tokenizer, model): self.tokenizer = tokenizer self.model = model # Ignore the very large sentinel used by tokenizers without a limit. limits = [ value for value in ( getattr(model.config, "max_position_embeddings", None), getattr(tokenizer, "model_max_length", None), ) if isinstance(value, int) and 0 < value < 1_000_000 ] self.context_limit = min(limits) if limits else 4096 def stream(self, message, history, system_prompt, max_new_tokens, temperature, top_p): if not message.strip(): raise gr.Error("Enter a message first.") messages = [] if system_prompt.strip(): messages.append({"role": "system", "content": system_prompt.strip()}) messages.extend( {"role": item["role"], "content": item["content"]} for item in history if item["role"] in ("user", "assistant") and isinstance(item["content"], str) ) messages.append({"role": "user", "content": message}) try: prompt = self.tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True ) except (ValueError, TypeError, KeyError) as exc: raise gr.Error( "Could not format this conversation with the model's chat template. " "Try an empty system prompt. If that fails, check the tokenizer's " "chat_template in the model repository." ) from exc # The template already supplies any required special tokens. inputs = self.tokenizer(prompt, return_tensors="pt", add_special_tokens=False) max_new_tokens = int(max_new_tokens) if inputs.input_ids.shape[-1] + max_new_tokens > self.context_limit: raise gr.Error( f"This conversation plus the requested reply exceeds the " f"{self.context_limit:,}-token context. Clear the chat, shorten your " "message, or reduce Max new tokens." ) inputs = inputs.to(self.model.device) streamer = TextIteratorStreamer( self.tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=110 ) cancelled = Event() errors = [] options = dict( **inputs, streamer=streamer, max_new_tokens=max_new_tokens, max_time=100, do_sample=temperature > 0, stopping_criteria=StoppingCriteriaList([StopOnEvent(cancelled)]), ) pad_token_id = self.tokenizer.pad_token_id if pad_token_id is None: pad_token_id = self.tokenizer.eos_token_id if pad_token_id is not None: options["pad_token_id"] = pad_token_id if temperature > 0: options.update(temperature=float(temperature), top_p=float(top_p)) def generate(): try: # inference_mode is thread-local, so enter it in the worker. with torch.inference_mode(): self.model.generate(**options) except Exception as exc: errors.append(exc) streamer.end() # Wake the consumer even if generate() fails. worker = Thread(target=generate, daemon=True) worker.start() try: response = "" for chunk in streamer: response += chunk yield response if errors: raise errors[0] except GeneratorExit: raise except Exception as exc: LOGGER.exception("Generation failed") raise gr.Error("Generation failed. Check the Space logs for details.") from exc finally: cancelled.set() # Do not leave generation using the GPU after the request releases it. worker.join() def load_model(): token = os.getenv("HF_TOKEN") if not token: raise RuntimeError( "Add HF_TOKEN as a Space secret with read access to " + MODEL_ID + "." ) common = dict(token=token, revision=MODEL_REVISION, trust_remote_code=True) tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, **common) device = "cuda" if torch.cuda.is_available() else "cpu" model = AutoModelForCausalLM.from_pretrained( MODEL_ID, **common, dtype=torch.bfloat16 if device == "cuda" else torch.float32, ).to(device).eval() return ChatModel(tokenizer, model) def create_demo(chat_model): @spaces.GPU(duration=120) def respond(message, history, system_prompt, max_new_tokens, temperature, top_p): yield from chat_model.stream( message, history, system_prompt, max_new_tokens, temperature, top_p ) with gr.Blocks(title="KV1 · Chat", theme=gr.themes.Soft()) as demo: gr.Markdown( "# KV1 · Chat\n" "Try [KV1 1.2B Prerelease](https://huggingface.co/" "klein-ml/KV1-1.2B-Prerelease). Responses stream with Markdown, " "tables, and syntax-highlighted code." ) with gr.Accordion("Generation settings", open=False): system_prompt = gr.Textbox( label="System prompt", value="", placeholder="Optional instructions…", lines=2, ) with gr.Row(): max_new_tokens = gr.Slider(32, 2048, value=512, step=32, label="Max new tokens") temperature = gr.Slider(0, 2, value=0.7, step=0.05, label="Temperature (0 = greedy)") top_p = gr.Slider(0.05, 1, value=0.9, step=0.05, label="Top-p") gr.ChatInterface( fn=respond, type="messages", chatbot=gr.Chatbot( type="messages", label="KV1", height=540, render_markdown=True, sanitize_html=True, show_copy_button=True, ), textbox=gr.Textbox( placeholder="Message KV1…", lines=2, max_lines=8, submit_btn="Send", stop_btn="Stop", ), additional_inputs=[system_prompt, max_new_tokens, temperature, top_p], examples=[ ["Explain how a rainbow forms.", "", 512, 0.7, 0.9], ["Write a Python function that checks whether a word is a palindrome.", "", 512, 0.7, 0.9], ["Compare lists and tuples in Python using a Markdown table.", "", 512, 0.7, 0.9], ], cache_examples=False, concurrency_limit=1, save_history=False, ) gr.Markdown("Prerelease model · Answers may be incorrect. · Powered by ZeroGPU") return demo.queue(max_size=20, default_concurrency_limit=1) if __name__ == "__main__": logging.basicConfig(level=logging.INFO) # Load and place the model at startup, outside the GPU-decorated callback. create_demo(load_model()).launch()