Spaces:
Running on Zero
Running on Zero
Download app.py from Banaxi-Tech/KV1-Demo: direct link, hf CLI and curl.
- Browser
- Download file 7.79 kB
-
https://huggingface.co/spaces/Banaxi-Tech/KV1-Demo/resolve/main/app.py
- Command line
-
hf download hf://spaces/Banaxi-Tech/KV1-Demo/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Banaxi-Tech/KV1-Demo/resolve/main/app.py
7.79 kB
| """KV1 chat demo for a Gradio Hugging Face ZeroGPU Space.""" | |
| import logging | |
| import os | |
| from threading import Event, Thread | |
| # Import spaces before torch so ZeroGPU can install its CUDA hooks. | |
| import spaces | |
| import gradio as gr | |
| import torch | |
| from transformers import ( | |
| AutoModelForCausalLM, | |
| AutoTokenizer, | |
| StoppingCriteria, | |
| StoppingCriteriaList, | |
| TextIteratorStreamer, | |
| ) | |
| MODEL_ID = "klein-ml/KV1-1.2B-Prerelease" | |
| MODEL_REVISION = os.getenv("MODEL_REVISION", "main") | |
| LOGGER = logging.getLogger(__name__) | |
| class StopOnEvent(StoppingCriteria): | |
| def __init__(self, event): | |
| self.event = event | |
| def __call__(self, input_ids, scores, **kwargs): | |
| return self.event.is_set() | |
| class ChatModel: | |
| def __init__(self, tokenizer, model): | |
| self.tokenizer = tokenizer | |
| self.model = model | |
| # Ignore the very large sentinel used by tokenizers without a limit. | |
| limits = [ | |
| value | |
| for value in ( | |
| getattr(model.config, "max_position_embeddings", None), | |
| getattr(tokenizer, "model_max_length", None), | |
| ) | |
| if isinstance(value, int) and 0 < value < 1_000_000 | |
| ] | |
| self.context_limit = min(limits) if limits else 4096 | |
| def stream(self, message, history, system_prompt, max_new_tokens, temperature, top_p): | |
| if not message.strip(): | |
| raise gr.Error("Enter a message first.") | |
| messages = [] | |
| if system_prompt.strip(): | |
| messages.append({"role": "system", "content": system_prompt.strip()}) | |
| messages.extend( | |
| {"role": item["role"], "content": item["content"]} | |
| for item in history | |
| if item["role"] in ("user", "assistant") | |
| and isinstance(item["content"], str) | |
| ) | |
| messages.append({"role": "user", "content": message}) | |
| try: | |
| prompt = self.tokenizer.apply_chat_template( | |
| messages, tokenize=False, add_generation_prompt=True | |
| ) | |
| except (ValueError, TypeError, KeyError) as exc: | |
| raise gr.Error( | |
| "Could not format this conversation with the model's chat template. " | |
| "Try an empty system prompt. If that fails, check the tokenizer's " | |
| "chat_template in the model repository." | |
| ) from exc | |
| # The template already supplies any required special tokens. | |
| inputs = self.tokenizer(prompt, return_tensors="pt", add_special_tokens=False) | |
| max_new_tokens = int(max_new_tokens) | |
| if inputs.input_ids.shape[-1] + max_new_tokens > self.context_limit: | |
| raise gr.Error( | |
| f"This conversation plus the requested reply exceeds the " | |
| f"{self.context_limit:,}-token context. Clear the chat, shorten your " | |
| "message, or reduce Max new tokens." | |
| ) | |
| inputs = inputs.to(self.model.device) | |
| streamer = TextIteratorStreamer( | |
| self.tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=110 | |
| ) | |
| cancelled = Event() | |
| errors = [] | |
| options = dict( | |
| **inputs, | |
| streamer=streamer, | |
| max_new_tokens=max_new_tokens, | |
| max_time=100, | |
| do_sample=temperature > 0, | |
| stopping_criteria=StoppingCriteriaList([StopOnEvent(cancelled)]), | |
| ) | |
| pad_token_id = self.tokenizer.pad_token_id | |
| if pad_token_id is None: | |
| pad_token_id = self.tokenizer.eos_token_id | |
| if pad_token_id is not None: | |
| options["pad_token_id"] = pad_token_id | |
| if temperature > 0: | |
| options.update(temperature=float(temperature), top_p=float(top_p)) | |
| def generate(): | |
| try: | |
| # inference_mode is thread-local, so enter it in the worker. | |
| with torch.inference_mode(): | |
| self.model.generate(**options) | |
| except Exception as exc: | |
| errors.append(exc) | |
| streamer.end() # Wake the consumer even if generate() fails. | |
| worker = Thread(target=generate, daemon=True) | |
| worker.start() | |
| try: | |
| response = "" | |
| for chunk in streamer: | |
| response += chunk | |
| yield response | |
| if errors: | |
| raise errors[0] | |
| except GeneratorExit: | |
| raise | |
| except Exception as exc: | |
| LOGGER.exception("Generation failed") | |
| raise gr.Error("Generation failed. Check the Space logs for details.") from exc | |
| finally: | |
| cancelled.set() | |
| # Do not leave generation using the GPU after the request releases it. | |
| worker.join() | |
| def load_model(): | |
| token = os.getenv("HF_TOKEN") | |
| if not token: | |
| raise RuntimeError( | |
| "Add HF_TOKEN as a Space secret with read access to " + MODEL_ID + "." | |
| ) | |
| common = dict(token=token, revision=MODEL_REVISION, trust_remote_code=True) | |
| tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, **common) | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| model = AutoModelForCausalLM.from_pretrained( | |
| MODEL_ID, | |
| **common, | |
| dtype=torch.bfloat16 if device == "cuda" else torch.float32, | |
| ).to(device).eval() | |
| return ChatModel(tokenizer, model) | |
| def create_demo(chat_model): | |
| def respond(message, history, system_prompt, max_new_tokens, temperature, top_p): | |
| yield from chat_model.stream( | |
| message, history, system_prompt, max_new_tokens, temperature, top_p | |
| ) | |
| with gr.Blocks(title="KV1 · Chat", theme=gr.themes.Soft()) as demo: | |
| gr.Markdown( | |
| "# KV1 · Chat\n" | |
| "Try [KV1 1.2B Prerelease](https://huggingface.co/" | |
| "klein-ml/KV1-1.2B-Prerelease). Responses stream with Markdown, " | |
| "tables, and syntax-highlighted code." | |
| ) | |
| with gr.Accordion("Generation settings", open=False): | |
| system_prompt = gr.Textbox( | |
| label="System prompt", value="", placeholder="Optional instructions…", | |
| lines=2, | |
| ) | |
| with gr.Row(): | |
| max_new_tokens = gr.Slider(32, 2048, value=512, step=32, label="Max new tokens") | |
| temperature = gr.Slider(0, 2, value=0.7, step=0.05, label="Temperature (0 = greedy)") | |
| top_p = gr.Slider(0.05, 1, value=0.9, step=0.05, label="Top-p") | |
| gr.ChatInterface( | |
| fn=respond, | |
| type="messages", | |
| chatbot=gr.Chatbot( | |
| type="messages", label="KV1", height=540, | |
| render_markdown=True, sanitize_html=True, | |
| show_copy_button=True, | |
| ), | |
| textbox=gr.Textbox( | |
| placeholder="Message KV1…", lines=2, max_lines=8, | |
| submit_btn="Send", stop_btn="Stop", | |
| ), | |
| additional_inputs=[system_prompt, max_new_tokens, temperature, top_p], | |
| examples=[ | |
| ["Explain how a rainbow forms.", "", 512, 0.7, 0.9], | |
| ["Write a Python function that checks whether a word is a palindrome.", "", 512, 0.7, 0.9], | |
| ["Compare lists and tuples in Python using a Markdown table.", "", 512, 0.7, 0.9], | |
| ], | |
| cache_examples=False, | |
| concurrency_limit=1, | |
| save_history=False, | |
| ) | |
| gr.Markdown("Prerelease model · Answers may be incorrect. · Powered by ZeroGPU") | |
| return demo.queue(max_size=20, default_concurrency_limit=1) | |
| if __name__ == "__main__": | |
| logging.basicConfig(level=logging.INFO) | |
| # Load and place the model at startup, outside the GPU-decorated callback. | |
| create_demo(load_model()).launch() | |