Add LoRA adapter, model card, and repointed base_model

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,97 @@
+---
+base_model: unsloth/Llama-3.2-3B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+license: llama3.2
+datasets:
+- Reubencf/frontend-react-dataset-no-images
+language:
+- en
+tags:
+- lora
+- peft
+- react
+- frontend
+- code-generation
+- llama-3.2
+---
+
+# Llama-3.2-3B React LoRA
+
+A LoRA adapter for **Llama 3.2 3B Instruct**, fine-tuned to generate React components
+from natural-language descriptions. Text-only — this variant was trained on the
+no-images split, so it does not take screenshots as input.
+
+## Usage
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+
+BASE = "unsloth/Llama-3.2-3B-Instruct"
+LORA = "Reubencf/llama-3.2-3b-react-lora"
+
+tok = AutoTokenizer.from_pretrained(BASE)
+model = AutoModelForCausalLM.from_pretrained(BASE, torch_dtype=torch.bfloat16, device_map="auto")
+model = PeftModel.from_pretrained(model, LORA)
+model.eval()
+
+msgs = [{"role": "user", "content": "Build a pricing table with three tiers and a monthly/yearly toggle."}]
+ids = tok.apply_chat_template(msgs, add_generation_prompt=True, return_tensors="pt").to(model.device)
+out = model.generate(ids, max_new_tokens=1024, temperature=0.7, top_p=0.9, do_sample=True)
+print(tok.decode(out[0][ids.shape[-1]:], skip_special_tokens=True))
+```
+
+Needs ~7 GB of VRAM in bf16. To merge the adapter into the base weights, call
+`model.merge_and_unload()`.
+
+## Training
+
+| | |
+|---|---|
+| Method | LoRA (PEFT 0.15.1) |
+| Rank / alpha / dropout | 64 / 128 / 0.05 |
+| Target modules | `q_proj` `k_proj` `v_proj` `o_proj` `gate_proj` `up_proj` `down_proj` |
+| Trainable params | 97.3 M across 392 tensors (stored fp32, 389 MB) |
+| Steps / epochs | 980 / 5, `train_batch_size=1` |
+| Data | [`Reubencf/frontend-react-dataset-no-images`](https://huggingface.co/datasets/Reubencf/frontend-react-dataset-no-images) |
+
+### Loss
+
+| Epoch | Eval loss |
+|---|---|
+| 1 | 0.556 |
+| 2 | 1.581 |
+| 3 | 0.541 |
+| 4 | 0.529 |
+| 5 | **0.525** |
+
+Train loss fell 1.15 → 0.62 over the run.
+
+## Limitations
+
+Read these before trusting the numbers above.
+
+- **The eval set is ~3 examples.** At that size the eval-loss column is close to
+  noise, and the epoch-2 spike to 1.58 is not interpretable. There is no held-out
+  benchmark and no human evaluation of the generated components.
+- **Probably trained on a subset.** 196 steps/epoch at batch size 1 accounts for
+  ~196 of the dataset's 1000 rows. `total_flos` (1.797e19) implies ~950 tokens per
+  example, which is consistent with that reading rather than with gradient
+  accumulation over the full set.
+- **Output is not validated.** Generated JSX is not compiled, linted, or rendered
+  during training or eval. Expect to fix imports and hallucinated component APIs.
+- **Small base model.** 3B params — weaker at multi-file work, state management,
+  and long components than larger code models.
+
+## Base model
+
+The adapter's `base_model_name_or_path` originally pointed at
+`togethercomputer/Meta-Llama-3.2-3B-Instruct-Reference__TOG__FT`, a Together
+training-internal reference that does not resolve on the Hub. It is repointed to
+`unsloth/Llama-3.2-3B-Instruct`, an ungated mirror of the same weights, so the
+adapter loads without a gate request. `meta-llama/Llama-3.2-3B-Instruct` is the
+canonical (manually gated) source and works identically if you have access.
+
+Licensed under the Llama 3.2 Community License, inherited from the base model.

adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "unsloth/Llama-3.2-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": [],
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05000000074505806,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "q_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "k_proj",
+    "up_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c5212462d657d59b6b76180cd064fa0b762ace4808e2be615ac751d3b8d7d13
+size 389074464

chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128009,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": 128009,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.13.0",
+  "use_cache": false,
+  "vocab_size": 128256,
+  "torch_dtype": "bfloat16"
+}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|begin_of_text|>",
+  "eos_token": "<|eot_id|>",
+  "pad_token": "<|eot_id|>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "padding_side": "right",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}