Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tekken.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,194 @@
+---
+library_name: transformers
+pipeline_tag: text-generation
+inference: true
+widget:
+  - text: Hello!
+    example_title: Hello world
+    group: Python
+base_model:
+- mistralai/Voxtral-Small-24B-2507
+---
+
+This tiny model is for debugging. It is randomly initialized with the config adapted from [mistralai/Voxtral-Small-24B-2507](https://huggingface.co/mistralai/Voxtral-Small-24B-2507).
+
+### Example usage:
+
+- vLLM
+
+```bash
+vllm serve yujiepan/voxtral-tiny-random --trust-remote-code
+```
+
+- Transformers
+
+```python
+import torch
+from transformers import AutoProcessor, VoxtralForConditionalGeneration
+
+model_id = "yujiepan/voxtral-tiny-random"
+
+device = "cuda"
+processor = AutoProcessor.from_pretrained(model_id)
+model = VoxtralForConditionalGeneration.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map=device)
+
+conversation = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "audio",
+                "path": "https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/mary_had_lamb.mp3",
+            },
+            {
+                "type": "audio",
+                "path": "https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/winning_call.mp3",
+            },
+            {"type": "text", "text": "What sport and what nursery rhyme are referenced?"},
+        ],
+    }
+]
+
+inputs = processor.apply_chat_template(conversation)
+inputs = inputs.to(device, dtype=torch.bfloat16)
+
+outputs = model.generate(**inputs, max_new_tokens=32)
+decoded_outputs = processor.batch_decode(outputs[:, inputs.input_ids.shape[1]:], skip_special_tokens=True)
+
+print("\nGenerated response:")
+print("=" * 80)
+print(decoded_outputs[0])
+print("=" * 80)
+```
+
+### Codes to create this repo:
+
+```python
+import json
+from pathlib import Path
+
+import accelerate
+import torch
+from huggingface_hub import file_exists, hf_hub_download
+from transformers import (
+    AutoConfig,
+    AutoModel,
+    AutoModelForCausalLM,
+    AutoProcessor,
+    GenerationConfig,
+    set_seed,
+)
+
+source_model_id = "mistralai/Voxtral-Small-24B-2507"
+save_folder = "/tmp/yujiepan/voxtral-tiny-random"
+
+processor = AutoProcessor.from_pretrained(source_model_id, trust_remote_code=True)
+processor.save_pretrained(save_folder)
+
+with open(hf_hub_download(source_model_id, filename='config.json', repo_type='model'), 'r', encoding='utf-8') as f:
+    config_json = json.load(f)
+config_json['audio_config'].update(
+    {
+        "head_dim": 32,
+        "hidden_size": 64,
+        "intermediate_size": 256,
+        "num_attention_heads": 2,
+        "num_key_value_heads": 2,
+        "num_hidden_layers": 2,
+    }
+)
+config_json['hidden_size'] = 64
+config_json['text_config'].update(
+    {
+        "head_dim": 32,
+        "hidden_size": 64,
+        "intermediate_size": 128,
+        "num_attention_heads": 2,
+        "num_key_value_heads": 1,
+        "num_hidden_layers": 2,
+        'tie_word_embeddings': True,
+    }
+)
+with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f:
+    json.dump(config_json, f, indent=2)
+config = AutoConfig.from_pretrained(
+    save_folder,
+    trust_remote_code=True,
+)
+print(config)
+torch.set_default_dtype(torch.bfloat16)
+model = AutoModel.from_config(config)
+torch.set_default_dtype(torch.float32)
+if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'):
+    model.generation_config = GenerationConfig.from_pretrained(
+        source_model_id, trust_remote_code=True,
+    )
+set_seed(42)
+model = model.cpu()  # cpu is more stable for random initialization across machines
+with torch.no_grad():
+    for name, p in sorted(model.named_parameters()):
+        torch.nn.init.normal_(p, 0, 0.2)
+        print(name, p.shape)
+model.save_pretrained(save_folder)
+print(model)
+```
+
+### Printing the model:
+
+```text
+VoxtralForConditionalGeneration(
+  (audio_tower): VoxtralEncoder(
+    (conv1): Conv1d(128, 64, kernel_size=(3,), stride=(1,), padding=(1,))
+    (conv2): Conv1d(64, 64, kernel_size=(3,), stride=(2,), padding=(1,))
+    (embed_positions): Embedding(1500, 64)
+    (layers): ModuleList(
+      (0-1): 2 x VoxtralEncoderLayer(
+        (self_attn): VoxtralAttention(
+          (k_proj): Linear(in_features=64, out_features=64, bias=False)
+          (v_proj): Linear(in_features=64, out_features=64, bias=True)
+          (q_proj): Linear(in_features=64, out_features=64, bias=True)
+          (out_proj): Linear(in_features=64, out_features=64, bias=True)
+        )
+        (self_attn_layer_norm): LayerNorm((64,), eps=1e-05, elementwise_affine=True)
+        (activation_fn): GELUActivation()
+        (fc1): Linear(in_features=64, out_features=256, bias=True)
+        (fc2): Linear(in_features=256, out_features=64, bias=True)
+        (final_layer_norm): LayerNorm((64,), eps=1e-05, elementwise_affine=True)
+      )
+    )
+    (layer_norm): LayerNorm((64,), eps=1e-05, elementwise_affine=True)
+    (avg_pooler): AvgPool1d(kernel_size=(2,), stride=(2,), padding=(0,))
+  )
+  (language_model): LlamaForCausalLM(
+    (model): LlamaModel(
+      (embed_tokens): Embedding(131072, 64)
+      (layers): ModuleList(
+        (0-1): 2 x LlamaDecoderLayer(
+          (self_attn): LlamaAttention(
+            (q_proj): Linear(in_features=64, out_features=64, bias=False)
+            (k_proj): Linear(in_features=64, out_features=32, bias=False)
+            (v_proj): Linear(in_features=64, out_features=32, bias=False)
+            (o_proj): Linear(in_features=64, out_features=64, bias=False)
+          )
+          (mlp): LlamaMLP(
+            (gate_proj): Linear(in_features=64, out_features=128, bias=False)
+            (up_proj): Linear(in_features=64, out_features=128, bias=False)
+            (down_proj): Linear(in_features=128, out_features=64, bias=False)
+            (act_fn): SiLU()
+          )
+          (input_layernorm): LlamaRMSNorm((64,), eps=1e-05)
+          (post_attention_layernorm): LlamaRMSNorm((64,), eps=1e-05)
+        )
+      )
+      (norm): LlamaRMSNorm((64,), eps=1e-05)
+      (rotary_emb): LlamaRotaryEmbedding()
+    )
+    (lm_head): Linear(in_features=64, out_features=131072, bias=False)
+  )
+  (multi_modal_projector): VoxtralMultiModalProjector(
+    (linear_1): Linear(in_features=256, out_features=64, bias=False)
+    (act): GELUActivation()
+    (linear_2): Linear(in_features=64, out_features=64, bias=False)
+  )
+)
+```

config.json

@@ -0,0 +1,54 @@
+{
+  "architectures": [
+    "VoxtralForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "attention_dropout": 0.0,
+    "dropout": 0.0,
+    "head_dim": 32,
+    "hidden_size": 64,
+    "initializer_range": 0.02,
+    "intermediate_size": 256,
+    "layerdrop": 0.0,
+    "max_source_positions": 1500,
+    "model_type": "voxtral_encoder",
+    "num_attention_heads": 2,
+    "num_hidden_layers": 2,
+    "num_key_value_heads": 2,
+    "num_mel_bins": 128,
+    "scale_embedding": false,
+    "vocab_size": 51866
+  },
+  "audio_token_id": 24,
+  "hidden_size": 64,
+  "model_type": "voxtral",
+  "projector_hidden_act": "gelu",
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "head_dim": 32,
+    "hidden_act": "silu",
+    "hidden_size": 64,
+    "initializer_range": 0.02,
+    "intermediate_size": 128,
+    "max_position_embeddings": 131072,
+    "mlp_bias": false,
+    "model_type": "llama",
+    "num_attention_heads": 2,
+    "num_hidden_layers": 2,
+    "num_key_value_heads": 1,
+    "pretraining_tp": 1,
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": null,
+    "rope_theta": 100000000.0,
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "vocab_size": 131072
+  },
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.54.0.dev0",
+  "vocab_size": 131072
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 11,
+  "transformers_version": "4.54.0.dev0",
+  "trust_remote_code": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbc8b2aa125d18ced83514cca0b8f156c6713acd05810f2a9b56e4018711483d
+size 17438688

preprocessor_config.json

@@ -0,0 +1,15 @@
+{
+  "chunk_length": 30,
+  "dither": 0.0,
+  "feature_extractor_type": "WhisperFeatureExtractor",
+  "feature_size": 128,
+  "hop_length": 160,
+  "n_fft": 400,
+  "n_samples": 480000,
+  "nb_max_frames": 3000,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "processor_class": "VoxtralProcessor",
+  "return_attention_mask": false,
+  "sampling_rate": 16000
+}

tekken.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4aaf3836c2a5332f029ce85a7a62255c966f47b6797ef81dedd0ade9c862e4a8
+size 14894206