Upload folder using huggingface_hub

checkpoint-10800/config.json

@@ -0,0 +1,44 @@
+{
+  "architectures": [
+    "UltravoxModel"
+  ],
+  "audio_latency_block_size": null,
+  "audio_model_id": "openai/whisper-large-v3-turbo",
+  "audio_model_lora_config": {
+    "lora_alpha": 8,
+    "r": 0,
+    "target_modules": [
+      "k_proj",
+      "q_proj",
+      "linear_k",
+      "linear_q"
+    ]
+  },
+  "auto_map": {
+    "AutoConfig": "ultravox_config.UltravoxConfig",
+    "AutoModel": "ultravox_model.UltravoxModel"
+  },
+  "hidden_size": 4096,
+  "ignore_index": -100,
+  "initializer_range": 0.02,
+  "model_type": "ultravox",
+  "norm_init": 0.4,
+  "pad_token_id": 2,
+  "projector_act": "swiglu",
+  "projector_ln_mid": false,
+  "stack_factor": 8,
+  "text_model_id": "HuggingFaceTB/SmolLM2-1.7B-Instruct",
+  "text_model_lora_config": {
+    "lora_alpha": 8,
+    "r": 0,
+    "target_modules": [
+      "k_proj",
+      "q_proj",
+      "linear_k",
+      "linear_q"
+    ]
+  },
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.47.0",
+  "vocab_size": 49152
+}

checkpoint-10800/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 2,
+  "transformers_version": "4.47.0"
+}

checkpoint-10800/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:599403a4e51e9462dec7603c72d5f6f03b3a9b3f316f6ecda2c082e6b9885566
+size 92299736

checkpoint-10800/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:802dc6c7082cb34cf728621561091d75e2351c60ad7abcbb07c6055f7cf2433e
+size 184602962

checkpoint-10800/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d902b2fc21ae01a2e62068637a291fad3a067be62ae6dd561ab05d3c42c1f44d
+size 14244

checkpoint-10800/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01ae122cbb1630c5e9128a1def009fb9fe6db25bf8dbd0de4ec680be8f1568e1
+size 1064

checkpoint-10800/special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|im_end|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-10800/tokenizer_config.json

@@ -0,0 +1,155 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|im_end|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

checkpoint-10800/trainer_state.json

@@ -0,0 +1,886 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.08988464803502172,
+  "eval_steps": 1000,
+  "global_step": 10800,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 8.322652595835345e-06,
+      "grad_norm": 14.9375,
+      "learning_rate": 2e-06,
+      "loss": 0.6484,
+      "step": 1
+    },
+    {
+      "epoch": 0.0008322652595835345,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.0002,
+      "loss": 0.4345,
+      "step": 100
+    },
+    {
+      "epoch": 0.001664530519167069,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 0.0004,
+      "loss": 0.2537,
+      "step": 200
+    },
+    {
+      "epoch": 0.0024967957787506035,
+      "grad_norm": 0.55859375,
+      "learning_rate": 0.0006,
+      "loss": 0.2344,
+      "step": 300
+    },
+    {
+      "epoch": 0.003329061038334138,
+      "grad_norm": 0.154296875,
+      "learning_rate": 0.0008,
+      "loss": 0.222,
+      "step": 400
+    },
+    {
+      "epoch": 0.004161326297917672,
+      "grad_norm": 0.126953125,
+      "learning_rate": 0.001,
+      "loss": 0.2142,
+      "step": 500
+    },
+    {
+      "epoch": 0.004993591557501207,
+      "grad_norm": 0.12353515625,
+      "learning_rate": 0.0012,
+      "loss": 0.2011,
+      "step": 600
+    },
+    {
+      "epoch": 0.005825856817084741,
+      "grad_norm": 0.10693359375,
+      "learning_rate": 0.0014,
+      "loss": 0.1897,
+      "step": 700
+    },
+    {
+      "epoch": 0.006658122076668276,
+      "grad_norm": 0.10595703125,
+      "learning_rate": 0.0016,
+      "loss": 0.1775,
+      "step": 800
+    },
+    {
+      "epoch": 0.0074903873362518105,
+      "grad_norm": 0.0859375,
+      "learning_rate": 0.0018000000000000002,
+      "loss": 0.1615,
+      "step": 900
+    },
+    {
+      "epoch": 0.008322652595835344,
+      "grad_norm": 0.083984375,
+      "learning_rate": 0.002,
+      "loss": 0.1444,
+      "step": 1000
+    },
+    {
+      "epoch": 0.008322652595835344,
+      "eval_peoplespeech-clean-transcription_loss": 2.2110702991485596,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.5405,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.708,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.839,
+      "step": 1000
+    },
+    {
+      "epoch": 0.009154917855418878,
+      "grad_norm": 0.0771484375,
+      "learning_rate": 0.001999725185109816,
+      "loss": 0.1323,
+      "step": 1100
+    },
+    {
+      "epoch": 0.009987183115002414,
+      "grad_norm": 0.083984375,
+      "learning_rate": 0.0019989008914857113,
+      "loss": 0.1242,
+      "step": 1200
+    },
+    {
+      "epoch": 0.010819448374585948,
+      "grad_norm": 0.07861328125,
+      "learning_rate": 0.00199752757218401,
+      "loss": 0.1162,
+      "step": 1300
+    },
+    {
+      "epoch": 0.011651713634169482,
+      "grad_norm": 0.07275390625,
+      "learning_rate": 0.001995605982021898,
+      "loss": 0.1128,
+      "step": 1400
+    },
+    {
+      "epoch": 0.012483978893753018,
+      "grad_norm": 0.07666015625,
+      "learning_rate": 0.0019931371771625545,
+      "loss": 0.1094,
+      "step": 1500
+    },
+    {
+      "epoch": 0.013316244153336551,
+      "grad_norm": 0.06201171875,
+      "learning_rate": 0.001990122514534651,
+      "loss": 0.1052,
+      "step": 1600
+    },
+    {
+      "epoch": 0.014148509412920085,
+      "grad_norm": 0.058837890625,
+      "learning_rate": 0.0019865636510865464,
+      "loss": 0.1022,
+      "step": 1700
+    },
+    {
+      "epoch": 0.014980774672503621,
+      "grad_norm": 0.0625,
+      "learning_rate": 0.001982462542875576,
+      "loss": 0.1011,
+      "step": 1800
+    },
+    {
+      "epoch": 0.015813039932087155,
+      "grad_norm": 0.06787109375,
+      "learning_rate": 0.001977821443992945,
+      "loss": 0.0983,
+      "step": 1900
+    },
+    {
+      "epoch": 0.01664530519167069,
+      "grad_norm": 0.0537109375,
+      "learning_rate": 0.001972642905324813,
+      "loss": 0.0975,
+      "step": 2000
+    },
+    {
+      "epoch": 0.01664530519167069,
+      "eval_peoplespeech-clean-transcription_loss": 1.6914767026901245,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.5301,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.716,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.839,
+      "step": 2000
+    },
+    {
+      "epoch": 0.017477570451254223,
+      "grad_norm": 0.05859375,
+      "learning_rate": 0.0019669297731502505,
+      "loss": 0.0947,
+      "step": 2100
+    },
+    {
+      "epoch": 0.018309835710837757,
+      "grad_norm": 0.062255859375,
+      "learning_rate": 0.00196068518757684,
+      "loss": 0.0935,
+      "step": 2200
+    },
+    {
+      "epoch": 0.019142100970421294,
+      "grad_norm": 0.059326171875,
+      "learning_rate": 0.001953912580814779,
+      "loss": 0.0911,
+      "step": 2300
+    },
+    {
+      "epoch": 0.019974366230004828,
+      "grad_norm": 0.060302734375,
+      "learning_rate": 0.0019466156752904343,
+      "loss": 0.0904,
+      "step": 2400
+    },
+    {
+      "epoch": 0.020806631489588362,
+      "grad_norm": 0.0615234375,
+      "learning_rate": 0.0019387984816003866,
+      "loss": 0.0882,
+      "step": 2500
+    },
+    {
+      "epoch": 0.021638896749171896,
+      "grad_norm": 0.056884765625,
+      "learning_rate": 0.0019304652963070869,
+      "loss": 0.0874,
+      "step": 2600
+    },
+    {
+      "epoch": 0.02247116200875543,
+      "grad_norm": 0.049072265625,
+      "learning_rate": 0.0019216206995773372,
+      "loss": 0.0871,
+      "step": 2700
+    },
+    {
+      "epoch": 0.023303427268338964,
+      "grad_norm": 0.053955078125,
+      "learning_rate": 0.0019122695526648968,
+      "loss": 0.0856,
+      "step": 2800
+    },
+    {
+      "epoch": 0.0241356925279225,
+      "grad_norm": 0.05517578125,
+      "learning_rate": 0.0019024169952385887,
+      "loss": 0.0845,
+      "step": 2900
+    },
+    {
+      "epoch": 0.024967957787506035,
+      "grad_norm": 0.0546875,
+      "learning_rate": 0.0018920684425573864,
+      "loss": 0.0852,
+      "step": 3000
+    },
+    {
+      "epoch": 0.024967957787506035,
+      "eval_peoplespeech-clean-transcription_loss": 1.6414048671722412,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.8151,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.521,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.815,
+      "step": 3000
+    },
+    {
+      "epoch": 0.02580022304708957,
+      "grad_norm": 0.052978515625,
+      "learning_rate": 0.0018812295824940284,
+      "loss": 0.0836,
+      "step": 3100
+    },
+    {
+      "epoch": 0.026632488306673103,
+      "grad_norm": 0.05126953125,
+      "learning_rate": 0.0018699063724087904,
+      "loss": 0.0821,
+      "step": 3200
+    },
+    {
+      "epoch": 0.027464753566256637,
+      "grad_norm": 0.056884765625,
+      "learning_rate": 0.0018581050358751443,
+      "loss": 0.0816,
+      "step": 3300
+    },
+    {
+      "epoch": 0.02829701882584017,
+      "grad_norm": 0.045654296875,
+      "learning_rate": 0.0018458320592590974,
+      "loss": 0.0814,
+      "step": 3400
+    },
+    {
+      "epoch": 0.029129284085423705,
+      "grad_norm": 0.04638671875,
+      "learning_rate": 0.0018330941881540914,
+      "loss": 0.0791,
+      "step": 3500
+    },
+    {
+      "epoch": 0.029961549345007242,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 0.0018198984236734246,
+      "loss": 0.0804,
+      "step": 3600
+    },
+    {
+      "epoch": 0.030793814604590776,
+      "grad_norm": 0.054931640625,
+      "learning_rate": 0.0018062520186022297,
+      "loss": 0.0802,
+      "step": 3700
+    },
+    {
+      "epoch": 0.03162607986417431,
+      "grad_norm": 0.054931640625,
+      "learning_rate": 0.0017921624734111292,
+      "loss": 0.0805,
+      "step": 3800
+    },
+    {
+      "epoch": 0.032458345123757844,
+      "grad_norm": 0.048095703125,
+      "learning_rate": 0.001777637532133752,
+      "loss": 0.079,
+      "step": 3900
+    },
+    {
+      "epoch": 0.03329061038334138,
+      "grad_norm": 0.0458984375,
+      "learning_rate": 0.0017626851781103819,
+      "loss": 0.0759,
+      "step": 4000
+    },
+    {
+      "epoch": 0.03329061038334138,
+      "eval_peoplespeech-clean-transcription_loss": 1.5952188968658447,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.4647,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.762,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.845,
+      "step": 4000
+    },
+    {
+      "epoch": 0.03412287564292491,
+      "grad_norm": 0.06494140625,
+      "learning_rate": 0.001747313629600077,
+      "loss": 0.0774,
+      "step": 4100
+    },
+    {
+      "epoch": 0.034955140902508446,
+      "grad_norm": 0.043701171875,
+      "learning_rate": 0.001731531335263669,
+      "loss": 0.0774,
+      "step": 4200
+    },
+    {
+      "epoch": 0.03578740616209198,
+      "grad_norm": 0.048095703125,
+      "learning_rate": 0.0017153469695201276,
+      "loss": 0.0769,
+      "step": 4300
+    },
+    {
+      "epoch": 0.03661967142167551,
+      "grad_norm": 0.051513671875,
+      "learning_rate": 0.0016987694277788418,
+      "loss": 0.0763,
+      "step": 4400
+    },
+    {
+      "epoch": 0.037451936681259054,
+      "grad_norm": 0.047607421875,
+      "learning_rate": 0.001681807821550438,
+      "loss": 0.0774,
+      "step": 4500
+    },
+    {
+      "epoch": 0.03828420194084259,
+      "grad_norm": 0.040283203125,
+      "learning_rate": 0.0016644714734388218,
+      "loss": 0.0748,
+      "step": 4600
+    },
+    {
+      "epoch": 0.03911646720042612,
+      "grad_norm": 0.05419921875,
+      "learning_rate": 0.0016467699120171987,
+      "loss": 0.0755,
+      "step": 4700
+    },
+    {
+      "epoch": 0.039948732460009656,
+      "grad_norm": 0.048828125,
+      "learning_rate": 0.001628712866590885,
+      "loss": 0.0732,
+      "step": 4800
+    },
+    {
+      "epoch": 0.04078099771959319,
+      "grad_norm": 0.046875,
+      "learning_rate": 0.0016103102618497923,
+      "loss": 0.0757,
+      "step": 4900
+    },
+    {
+      "epoch": 0.041613262979176724,
+      "grad_norm": 0.042236328125,
+      "learning_rate": 0.0015915722124135226,
+      "loss": 0.0738,
+      "step": 5000
+    },
+    {
+      "epoch": 0.041613262979176724,
+      "eval_peoplespeech-clean-transcription_loss": 1.5546748638153076,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.345,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.849,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.856,
+      "step": 5000
+    },
+    {
+      "epoch": 0.04244552823876026,
+      "grad_norm": 0.045166015625,
+      "learning_rate": 0.001572509017272072,
+      "loss": 0.0742,
+      "step": 5100
+    },
+    {
+      "epoch": 0.04327779349834379,
+      "grad_norm": 0.0556640625,
+      "learning_rate": 0.0015531311541251993,
+      "loss": 0.0746,
+      "step": 5200
+    },
+    {
+      "epoch": 0.044110058757927326,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 0.0015334492736235703,
+      "loss": 0.073,
+      "step": 5300
+    },
+    {
+      "epoch": 0.04494232401751086,
+      "grad_norm": 0.044921875,
+      "learning_rate": 0.0015134741935148419,
+      "loss": 0.0727,
+      "step": 5400
+    },
+    {
+      "epoch": 0.045774589277094394,
+      "grad_norm": 0.05029296875,
+      "learning_rate": 0.0014932168926979072,
+      "loss": 0.0726,
+      "step": 5500
+    },
+    {
+      "epoch": 0.04660685453667793,
+      "grad_norm": 0.048095703125,
+      "learning_rate": 0.0014726885051885652,
+      "loss": 0.0718,
+      "step": 5600
+    },
+    {
+      "epoch": 0.04743911979626146,
+      "grad_norm": 0.046875,
+      "learning_rate": 0.0014519003139999338,
+      "loss": 0.0736,
+      "step": 5700
+    },
+    {
+      "epoch": 0.048271385055845,
+      "grad_norm": 0.051513671875,
+      "learning_rate": 0.0014308637449409706,
+      "loss": 0.0724,
+      "step": 5800
+    },
+    {
+      "epoch": 0.049103650315428536,
+      "grad_norm": 0.053955078125,
+      "learning_rate": 0.0014095903603365066,
+      "loss": 0.0724,
+      "step": 5900
+    },
+    {
+      "epoch": 0.04993591557501207,
+      "grad_norm": 0.037841796875,
+      "learning_rate": 0.0013880918526722496,
+      "loss": 0.071,
+      "step": 6000
+    },
+    {
+      "epoch": 0.04993591557501207,
+      "eval_peoplespeech-clean-transcription_loss": 1.5297455787658691,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.3364,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.855,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.857,
+      "step": 6000
+    },
+    {
+      "epoch": 0.050768180834595604,
+      "grad_norm": 0.05029296875,
+      "learning_rate": 0.0013663800381682463,
+      "loss": 0.0713,
+      "step": 6100
+    },
+    {
+      "epoch": 0.05160044609417914,
+      "grad_norm": 0.037109375,
+      "learning_rate": 0.0013444668502843329,
+      "loss": 0.0701,
+      "step": 6200
+    },
+    {
+      "epoch": 0.05243271135376267,
+      "grad_norm": 0.03564453125,
+      "learning_rate": 0.0013223643331611537,
+      "loss": 0.0708,
+      "step": 6300
+    },
+    {
+      "epoch": 0.053264976613346206,
+      "grad_norm": 0.04443359375,
+      "learning_rate": 0.001300084635000341,
+      "loss": 0.0707,
+      "step": 6400
+    },
+    {
+      "epoch": 0.05409724187292974,
+      "grad_norm": 0.033935546875,
+      "learning_rate": 0.0012776400013875004,
+      "loss": 0.0704,
+      "step": 6500
+    },
+    {
+      "epoch": 0.054929507132513274,
+      "grad_norm": 0.046142578125,
+      "learning_rate": 0.0012550427685616766,
+      "loss": 0.0707,
+      "step": 6600
+    },
+    {
+      "epoch": 0.05576177239209681,
+      "grad_norm": 0.04150390625,
+      "learning_rate": 0.0012323053566349834,
+      "loss": 0.0696,
+      "step": 6700
+    },
+    {
+      "epoch": 0.05659403765168034,
+      "grad_norm": 0.042236328125,
+      "learning_rate": 0.0012094402627661448,
+      "loss": 0.0705,
+      "step": 6800
+    },
+    {
+      "epoch": 0.057426302911263875,
+      "grad_norm": 0.035888671875,
+      "learning_rate": 0.0011864600542916813,
+      "loss": 0.0686,
+      "step": 6900
+    },
+    {
+      "epoch": 0.05825856817084741,
+      "grad_norm": 0.0498046875,
+      "learning_rate": 0.0011633773618185302,
+      "loss": 0.0687,
+      "step": 7000
+    },
+    {
+      "epoch": 0.05825856817084741,
+      "eval_peoplespeech-clean-transcription_loss": 1.518404483795166,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.4703,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.758,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.845,
+      "step": 7000
+    },
+    {
+      "epoch": 0.05909083343043095,
+      "grad_norm": 0.034423828125,
+      "learning_rate": 0.0011402048722818862,
+      "loss": 0.0693,
+      "step": 7100
+    },
+    {
+      "epoch": 0.059923098690014484,
+      "grad_norm": 0.047119140625,
+      "learning_rate": 0.0011169553219720827,
+      "loss": 0.0697,
+      "step": 7200
+    },
+    {
+      "epoch": 0.06075536394959802,
+      "grad_norm": 0.0576171875,
+      "learning_rate": 0.001093641489534351,
+      "loss": 0.0685,
+      "step": 7300
+    },
+    {
+      "epoch": 0.06158762920918155,
+      "grad_norm": 0.052490234375,
+      "learning_rate": 0.001070276188945293,
+      "loss": 0.0666,
+      "step": 7400
+    },
+    {
+      "epoch": 0.062419894468765086,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 0.00104687226246994,
+      "loss": 0.0692,
+      "step": 7500
+    },
+    {
+      "epoch": 0.06325215972834862,
+      "grad_norm": 0.04052734375,
+      "learning_rate": 0.0010234425736032607,
+      "loss": 0.069,
+      "step": 7600
+    },
+    {
+      "epoch": 0.06408442498793215,
+      "grad_norm": 0.06298828125,
+      "learning_rate": 0.001,
+      "loss": 0.0682,
+      "step": 7700
+    },
+    {
+      "epoch": 0.06491669024751569,
+      "grad_norm": 0.04736328125,
+      "learning_rate": 0.0009765574263967396,
+      "loss": 0.0676,
+      "step": 7800
+    },
+    {
+      "epoch": 0.06574895550709922,
+      "grad_norm": 0.064453125,
+      "learning_rate": 0.0009531277375300599,
+      "loss": 0.0685,
+      "step": 7900
+    },
+    {
+      "epoch": 0.06658122076668276,
+      "grad_norm": 0.037109375,
+      "learning_rate": 0.0009297238110547074,
+      "loss": 0.0677,
+      "step": 8000
+    },
+    {
+      "epoch": 0.06658122076668276,
+      "eval_peoplespeech-clean-transcription_loss": 1.5178765058517456,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.3361,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.855,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.857,
+      "step": 8000
+    },
+    {
+      "epoch": 0.06741348602626629,
+      "grad_norm": 0.041259765625,
+      "learning_rate": 0.0009063585104656494,
+      "loss": 0.068,
+      "step": 8100
+    },
+    {
+      "epoch": 0.06824575128584982,
+      "grad_norm": 0.0390625,
+      "learning_rate": 0.0008830446780279176,
+      "loss": 0.0681,
+      "step": 8200
+    },
+    {
+      "epoch": 0.06907801654543336,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 0.0008597951277181142,
+      "loss": 0.0677,
+      "step": 8300
+    },
+    {
+      "epoch": 0.06991028180501689,
+      "grad_norm": 0.054443359375,
+      "learning_rate": 0.0008366226381814697,
+      "loss": 0.0679,
+      "step": 8400
+    },
+    {
+      "epoch": 0.07074254706460043,
+      "grad_norm": 0.036865234375,
+      "learning_rate": 0.000813539945708319,
+      "loss": 0.0685,
+      "step": 8500
+    },
+    {
+      "epoch": 0.07157481232418396,
+      "grad_norm": 0.036865234375,
+      "learning_rate": 0.0007905597372338558,
+      "loss": 0.0678,
+      "step": 8600
+    },
+    {
+      "epoch": 0.07240707758376749,
+      "grad_norm": 0.03466796875,
+      "learning_rate": 0.0007676946433650169,
+      "loss": 0.0671,
+      "step": 8700
+    },
+    {
+      "epoch": 0.07323934284335103,
+      "grad_norm": 0.03857421875,
+      "learning_rate": 0.0007449572314383236,
+      "loss": 0.0659,
+      "step": 8800
+    },
+    {
+      "epoch": 0.07407160810293456,
+      "grad_norm": 0.04248046875,
+      "learning_rate": 0.0007223599986124993,
+      "loss": 0.0657,
+      "step": 8900
+    },
+    {
+      "epoch": 0.07490387336251811,
+      "grad_norm": 0.0517578125,
+      "learning_rate": 0.0006999153649996595,
+      "loss": 0.0662,
+      "step": 9000
+    },
+    {
+      "epoch": 0.07490387336251811,
+      "eval_peoplespeech-clean-transcription_loss": 1.49990975856781,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.6057,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.663,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.833,
+      "step": 9000
+    },
+    {
+      "epoch": 0.07573613862210164,
+      "grad_norm": 0.05419921875,
+      "learning_rate": 0.0006776356668388464,
+      "loss": 0.0659,
+      "step": 9100
+    },
+    {
+      "epoch": 0.07656840388168518,
+      "grad_norm": 0.041015625,
+      "learning_rate": 0.0006555331497156671,
+      "loss": 0.0656,
+      "step": 9200
+    },
+    {
+      "epoch": 0.07740066914126871,
+      "grad_norm": 0.08203125,
+      "learning_rate": 0.0006336199618317538,
+      "loss": 0.0663,
+      "step": 9300
+    },
+    {
+      "epoch": 0.07823293440085224,
+      "grad_norm": 0.03271484375,
+      "learning_rate": 0.0006119081473277501,
+      "loss": 0.0663,
+      "step": 9400
+    },
+    {
+      "epoch": 0.07906519966043578,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 0.0005904096396634935,
+      "loss": 0.0642,
+      "step": 9500
+    },
+    {
+      "epoch": 0.07989746492001931,
+      "grad_norm": 0.03271484375,
+      "learning_rate": 0.0005691362550590297,
+      "loss": 0.0644,
+      "step": 9600
+    },
+    {
+      "epoch": 0.08072973017960285,
+      "grad_norm": 0.047607421875,
+      "learning_rate": 0.0005480996860000663,
+      "loss": 0.0648,
+      "step": 9700
+    },
+    {
+      "epoch": 0.08156199543918638,
+      "grad_norm": 0.03955078125,
+      "learning_rate": 0.0005273114948114346,
+      "loss": 0.0655,
+      "step": 9800
+    },
+    {
+      "epoch": 0.08239426069876991,
+      "grad_norm": 0.048095703125,
+      "learning_rate": 0.0005067831073020928,
+      "loss": 0.0651,
+      "step": 9900
+    },
+    {
+      "epoch": 0.08322652595835345,
+      "grad_norm": 0.0299072265625,
+      "learning_rate": 0.00048652580648515787,
+      "loss": 0.066,
+      "step": 10000
+    },
+    {
+      "epoch": 0.08322652595835345,
+      "eval_peoplespeech-clean-transcription_loss": 1.489192247390747,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.4911,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.743,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.843,
+      "step": 10000
+    },
+    {
+      "epoch": 0.08405879121793698,
+      "grad_norm": 0.055908203125,
+      "learning_rate": 0.0004665507263764299,
+      "loss": 0.0653,
+      "step": 10100
+    },
+    {
+      "epoch": 0.08489105647752052,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 0.0004468688458748006,
+      "loss": 0.065,
+      "step": 10200
+    },
+    {
+      "epoch": 0.08572332173710405,
+      "grad_norm": 0.05322265625,
+      "learning_rate": 0.0004274909827279283,
+      "loss": 0.0654,
+      "step": 10300
+    },
+    {
+      "epoch": 0.08655558699668758,
+      "grad_norm": 0.0419921875,
+      "learning_rate": 0.0004084277875864776,
+      "loss": 0.0648,
+      "step": 10400
+    },
+    {
+      "epoch": 0.08738785225627112,
+      "grad_norm": 0.050537109375,
+      "learning_rate": 0.00038968973815020803,
+      "loss": 0.0643,
+      "step": 10500
+    },
+    {
+      "epoch": 0.08822011751585465,
+      "grad_norm": 0.0458984375,
+      "learning_rate": 0.00037128713340911534,
+      "loss": 0.0669,
+      "step": 10600
+    },
+    {
+      "epoch": 0.08905238277543819,
+      "grad_norm": 0.05029296875,
+      "learning_rate": 0.00035323008798280133,
+      "loss": 0.0657,
+      "step": 10700
+    },
+    {
+      "epoch": 0.08988464803502172,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 0.00033552852656117837,
+      "loss": 0.0646,
+      "step": 10800
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 14400,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 3600,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.93903470695465e+17,
+  "train_batch_size": 24,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-10800/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37b86db64b66888697120b24a102f1ab62ab1d7da7b6bd4cd2e497d4499c3fb5
+size 5688

checkpoint-10800/ultravox_config.py

@@ -0,0 +1,173 @@
+import dataclasses
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+import transformers
+
+
+@dataclasses.dataclass
+class LoraConfigSimplified:
+    """
+    Low Rank Approximation (LoRA) configuration.
+
+    Used for language and audio models separately.
+    """
+
+    # The rank of the approximation
+    r: int = 0
+    lora_alpha: float = 8
+    target_modules: Optional[List[str]] = dataclasses.field(
+        default_factory=lambda: ["k_proj", "q_proj", "linear_k", "linear_q"]
+    )
+    # A list of module names regex patterns to unfreeze. Only used if r == 0.
+    unfreeze_layers: Optional[List[str]] = None
+
+
+class LossFunction(str, Enum):
+    CrossEntropy = "ce"
+    KL_Divergence = "kl"
+
+
+@dataclasses.dataclass
+class LossConfig:
+    loss_function: LossFunction = LossFunction.CrossEntropy
+    kl_temperature: float = 2.0
+
+    @property
+    def requires_alt_fields(self):
+        return self.loss_function == LossFunction.KL_Divergence
+
+
+class UltravoxConfig(transformers.PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`UltravoxForConditionalGeneration`]. It is used to instantiate an
+    Ultravox model according to the specified arguments, defining the model architecture.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        audio_config (`Wav2Vec2Config`,  *optional*):
+            Custom audio config or dict
+        text_config (`Union[AutoConfig, dict]`, *optional*):
+            The config object of the text backbone. Can be any of `LlamaConfig` or `MistralConfig`.
+        ignore_index (`int`, *optional*, defaults to -100):
+            The ignore index for the loss function.
+        audio_token_index (`int`, *optional*, defaults to 32000):
+            The audio token index to encode the audio prompt.
+        stack_factor (`int`, *optional*, defaults to 8):
+            Audio downsampling factor for the multimodal projector.
+        norm_init (`float`, *optional*, defaults to 0.4):
+            The initialization value for the layer normalization.
+        projector_act (`str`, *optional*, defaults to `"swiglu"`):
+            The activation function used by the multimodal projector.
+        text_model_lora_config (`LoraConfigSimplified`, *optional*):
+            The LoRA configuration for finetuning the text model.
+        audio_model_lora_config (`LoraConfigSimplified`, *optional*):
+            The LoRA configuration for finetuning the audio model.
+        audio_latency_block_size (`int`, *optional*, defaults to `None`):
+            The latency block size for simulating audio streaming.
+
+
+    Example:
+
+    ```python
+    >>> from transformers import UltravoxModel, Wav2Vec2Config, UltravoxConfig, LlamaConfig
+
+    >>> # Initializing an audio encoder config
+    >>> audio_config = Wav2Vec2Config()
+
+    >>> # Initializing a Llama config
+    >>> text_config = LlamaConfig()
+
+    >>> # Initializing a default configuration
+    >>> configuration = UltravoxConfig(audio_config, text_config)
+
+    >>> # Initializing a completely untrained model from the configuration
+    >>> model = UltravoxModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+
+    >>> # Initialize a model from pretrained checkpoints and random projector weights
+    >>> config = UltravoxConfig(audio_model_id="facebook/wav2vec2-base-960h", text_model_id="meta-llama/Llama-2-7b-chat-hf")
+    ```"""
+
+    model_type = "ultravox"
+    is_composition = False
+
+    def __init__(
+        self,
+        audio_config: Optional[Dict[str, Any]] = None,
+        text_config: Optional[Dict[str, Any]] = None,
+        audio_model_id: Optional[str] = None,
+        text_model_id: Optional[str] = None,
+        ignore_index: int = -100,
+        hidden_size: int = 4096,
+        stack_factor: int = 8,
+        norm_init: float = 0.4,
+        projector_act: str = "swiglu",
+        projector_ln_mid: bool = False,  # defaults to False for compatibility with v0.4.1 and below
+        text_model_lora_config: Optional[LoraConfigSimplified] = None,
+        audio_model_lora_config: Optional[LoraConfigSimplified] = None,
+        audio_latency_block_size: Optional[int] = None,
+        **kwargs,
+    ):
+        self.ignore_index = ignore_index
+
+        self.audio_model_id = audio_model_id
+        self.text_model_id = text_model_id
+
+        self.hidden_size = hidden_size
+        self.stack_factor = stack_factor
+        self.norm_init = norm_init
+        self.projector_act = projector_act
+        self.projector_ln_mid = projector_ln_mid
+        if text_model_id is not None:
+            self.text_config: transformers.LlamaConfig = (
+                transformers.AutoConfig.from_pretrained(text_model_id)
+            )
+        else:
+            text_config = text_config or {}
+            self.text_config = transformers.CONFIG_MAPPING[
+                text_config.get("model_type", "llama")
+            ](**text_config)
+
+        if audio_model_id is not None:
+            self.audio_config: transformers.PretrainedConfig = (
+                transformers.AutoConfig.from_pretrained(audio_model_id)
+            )
+        else:
+            audio_config = audio_config or {}
+            self.audio_config = transformers.CONFIG_MAPPING[
+                audio_config.get("model_type", "wav2vec2")
+            ](**audio_config)
+
+        self.text_model_lora_config = (
+            text_model_lora_config
+            if isinstance(text_model_lora_config, dict)
+            else dataclasses.asdict(text_model_lora_config or LoraConfigSimplified())
+        )
+        self.audio_model_lora_config = (
+            audio_model_lora_config
+            if isinstance(audio_model_lora_config, dict)
+            else dataclasses.asdict(audio_model_lora_config or LoraConfigSimplified())
+        )
+        self.audio_latency_block_size = audio_latency_block_size
+
+        self.vocab_size = self.text_config.vocab_size
+
+        self.initializer_range = self.text_config.initializer_range
+
+        super().__init__(**kwargs)
+
+    def to_diff_dict(self) -> Dict[str, Any]:
+        diff_dict = super().to_diff_dict()
+
+        # remove text_config and audio_config if text_model_id and audio_model_id are present
+        if self.text_model_id is not None:
+            diff_dict.pop("text_config", None)
+        if self.audio_model_id is not None:
+            diff_dict.pop("audio_config", None)
+
+        return diff_dict

checkpoint-10800/ultravox_model.py

@@ -0,0 +1,754 @@
+import logging
+import re
+from typing import Any, Dict, Optional, Set, Tuple, Union
+
+import peft
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import transformers
+import transformers.activations
+import transformers.modeling_outputs
+import transformers.models
+from transformers.models.whisper import modeling_whisper as whisper
+
+# We must use relative import in this directory to allow uploading to HF Hub
+# Even "from . import X" pattern doesn't work (undocumented and unclear why)
+from .ultravox_config import LossConfig
+from .ultravox_config import LossFunction
+from .ultravox_config import UltravoxConfig
+
+
+class UltravoxModel(transformers.LlamaPreTrainedModel):
+    """
+    The Ultravox model which consists of an audio encoder and a language model.
+
+    Audio input is processed by the audio encoder, then every `stack_factor` frames are stacked together and
+    projected to the language model's embedding space using a few linear layers.
+    The text is embedded by the language model as usual and then the audio and text embeddings are merged together.
+
+    A special token `<|audio|>` is used to indicate the start of the audio embeddings in the merged embeddings.
+
+    Parameters:
+        config: Model configuration class with all the parameters of the model.
+    """
+
+    config_class = UltravoxConfig
+    config: UltravoxConfig  # for type hinting
+    # Usually we load encoder and LLM weights from a pretrained model separately, so they are allowed to be missing
+    _keys_to_ignore_on_load_missing = ["audio_tower.*", "language_model.*"]
+    # Since we have kwargs in forward, we need to set this to False, otherwise grad_accum_steps will cause incorrect train loss to be reported
+    # see https://github.com/huggingface/transformers/issues/35856 and https://github.com/huggingface/trl/pull/2615/files
+    accepts_loss_kwargs = False
+
+    def __init__(self, config: UltravoxConfig):
+        super().__init__(config)
+        self._register_load_state_dict_pre_hook(self._pre_load_state_dict_hook)
+
+        self.keep_params: Set[str] = set()
+        self.vocab_size = config.vocab_size
+
+        self.audio_tower = self._create_audio_tower(config)
+        self.multi_modal_projector = self._create_multi_modal_projector(config)
+        self.language_model = self._create_language_model(config)
+
+        # Determine no_split_modules dynamically to use with FSDP auto_wrap policy.
+        # FSDP throws an error if some of the layer types are not found in the model.
+        # This would be something like ["LlamaDecoderLayer", "WhisperEncoderLayer"]
+        self._no_split_modules = (self.language_model._no_split_modules or []) + (
+            self.audio_tower._no_split_modules or []
+        )
+
+        self.loss_config = LossConfig()
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.language_model.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.language_model.get_output_embeddings()
+
+    def set_output_embeddings(self, new_embeddings):
+        self.language_model.set_output_embeddings(new_embeddings)
+
+    def set_decoder(self, decoder):
+        self.language_model.set_decoder(decoder)
+
+    def get_decoder(self):
+        return self.language_model.get_decoder()
+
+    def tie_weights(self):
+        return self.language_model.tie_weights()
+
+    def set_loss_config(self, loss_config: LossConfig):
+        self.loss_config = loss_config
+
+    def _setup_cache(
+        self, cache_cls, max_batch_size: int, max_cache_len: Optional[int] = None
+    ):
+        self.language_model._setup_cache(cache_cls, max_batch_size, max_cache_len)
+
+    def _reorder_cache(self, past_key_values, beam_idx):
+        return self.language_model._reorder_cache(past_key_values, beam_idx)
+
+    def resize_token_embeddings(
+        self,
+        new_num_tokens: Optional[int] = None,
+        pad_to_multiple_of: Optional[int] = None,
+    ) -> nn.Embedding:
+        model_embeds = self.language_model.resize_token_embeddings(
+            new_num_tokens, pad_to_multiple_of
+        )
+        # update vocab size
+        self.config.text_config.vocab_size = model_embeds.num_embeddings
+        self.config.vocab_size = model_embeds.num_embeddings
+        self.vocab_size = model_embeds.num_embeddings
+        return model_embeds
+
+    def _compute_kl_loss(
+        self,
+        lm_output: transformers.modeling_outputs.CausalLMOutputWithPast,
+        labels: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]] = None,
+        alt_input_ids: Optional[torch.Tensor] = None,
+        alt_attention_mask: Optional[torch.Tensor] = None,
+        alt_labels: Optional[torch.Tensor] = None,
+        **kwargs,
+    ):
+        # disable gradient computation for the teacher model
+        with torch.no_grad():
+            # compute the teacher (text-only) model's distribution
+            alt_inputs_embeds = self.get_input_embeddings().forward(alt_input_ids)
+            alt_lm_output = self.language_model.forward(
+                inputs_embeds=alt_inputs_embeds,
+                labels=alt_labels,
+                attention_mask=alt_attention_mask,
+                past_key_values=past_key_values,
+                **kwargs,
+            )
+        # compute the KL divergence loss between the two models
+        kl_loss = F.kl_div(
+            F.log_softmax(
+                lm_output.logits[labels != -100] / self.loss_config.kl_temperature,
+                dim=-1,
+            ),
+            F.softmax(
+                alt_lm_output.logits[alt_labels != -100]
+                / self.loss_config.kl_temperature,
+                dim=-1,
+            ),
+            reduction="batchmean",
+        )
+        return {"loss": kl_loss}
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        audio_values: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        audio_token_start_idx: Optional[torch.Tensor] = None,
+        audio_len: Optional[torch.Tensor] = None,
+        audio_token_len: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]] = None,
+        # the alt_* fields are needed for KL divergence loss
+        alt_input_ids: Optional[torch.Tensor] = None,
+        alt_attention_mask: Optional[torch.Tensor] = None,
+        alt_labels: Optional[torch.Tensor] = None,
+        **kwargs,
+    ) -> Union[Tuple, transformers.modeling_outputs.CausalLMOutputWithPast]:
+        """
+        Forward pass for the Ultravox model.
+
+        `input_ids` are the tokenized text input. They are embedded by the language model as usual.
+        `audio_values` are processed by the audio encoder and then every `stack_factor` frames are stacked together and
+        projected to the language model's embedding space using a few linear layers.
+        The audio and text embeddings are merged together. A special token `<|audio|>` is used to indicate the start
+        of the audio embeddings in the merged embeddings.
+
+        Args:
+            input_ids: The tokenized text input.
+            audio_values: The processed audio values.
+            inputs_embeds: The embeddings for the input tokens.
+            labels: The tokenized text labels.
+            attention_mask: The attention mask for the input.
+            position_ids: The position ids for the input.
+            past_key_values: The past key value cache for the language model attention layers.
+            **kwargs: Additional keyword arguments. Passed directly to the language model.
+        """
+        if inputs_embeds is None:
+            # B x T  ->  B x T x D
+            inputs_embeds = self.get_input_embeddings().forward(input_ids)
+
+        if audio_values is not None:
+            assert (
+                audio_token_start_idx is not None and audio_token_len is not None
+            ), "audio_token_start_idx and audio_token_len must be provided if audio_values are provided."
+            assert (
+                len(audio_token_start_idx) == len(audio_token_len) == len(audio_values)
+            ), "audio_token_start_idx, audio_token_len, and audio_values must have the same batch size."
+
+            # B x A/3200 x D
+            audio_tower_output = self.audio_tower.forward(
+                audio_values.to(self.audio_tower.dtype),
+                audio_len=audio_len,
+            ).last_hidden_state
+            audio_tower_output = audio_tower_output.to(inputs_embeds.dtype)
+
+            audio_embeds = self.multi_modal_projector.forward(audio_tower_output)
+
+            # combine audio and text embeddings
+            for i, (audio, start, length) in enumerate(
+                zip(audio_embeds, audio_token_start_idx, audio_token_len)
+            ):
+                length = min(length, audio.shape[0])
+                inputs_embeds[i, start : start + length] = audio[:length]
+
+        lm_output = self.language_model.forward(
+            inputs_embeds=inputs_embeds,
+            labels=labels,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            **kwargs,
+        )
+        if self.training:
+            if self.loss_config.loss_function == LossFunction.CrossEntropy:
+                return lm_output
+            elif self.loss_config.loss_function == LossFunction.KL_Divergence:
+                return self._compute_kl_loss(
+                    lm_output=lm_output,
+                    labels=labels,
+                    past_key_values=past_key_values,
+                    alt_input_ids=alt_input_ids,
+                    alt_attention_mask=alt_attention_mask,
+                    alt_labels=alt_labels,
+                    **kwargs,
+                )
+            else:
+                raise ValueError(
+                    f"Unsupported loss function: {self.loss_config.loss_function}"
+                )
+        else:
+            return lm_output
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids: torch.Tensor,
+        audio_values: Optional[torch.FloatTensor] = None,
+        audio_token_start_idx: Optional[torch.Tensor] = None,
+        audio_token_len: Optional[torch.Tensor] = None,
+        audio_len: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        cache_position: Optional[torch.Tensor] = None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        model_input = self.language_model.prepare_inputs_for_generation(
+            input_ids=input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            inputs_embeds=inputs_embeds,
+            cache_position=cache_position,
+            **kwargs,
+        )
+
+        # include audio information in model_input only when it is needed during prefilling
+        # audio_token_start_idx should always be relative to the current cache position
+        prefill_start_idx = 0 if cache_position is None else cache_position[0]
+        if (
+            audio_values is not None
+            and audio_token_start_idx is not None
+            and prefill_start_idx <= torch.max(audio_token_start_idx)
+        ):
+            model_input["audio_values"] = audio_values
+            model_input["audio_token_start_idx"] = (
+                audio_token_start_idx - prefill_start_idx
+            )
+            model_input["audio_token_len"] = audio_token_len
+            model_input["audio_len"] = audio_len
+
+        return model_input
+
+    @classmethod
+    def _create_multi_modal_projector(
+        cls, config: UltravoxConfig
+    ) -> "UltravoxProjector":
+        projector = UltravoxProjector(config)
+        projector.to(config.torch_dtype)
+        return projector
+
+    @classmethod
+    def _create_audio_tower(
+        cls, config: UltravoxConfig
+    ) -> Union[transformers.Wav2Vec2Model, "ModifiedWhisperEncoder"]:
+        if config.audio_model_id is not None:
+            if "whisper" in config.audio_model_id.lower():
+                audio_tower = ModifiedWhisperEncoder.from_pretrained(
+                    config.audio_model_id, torch_dtype=config.torch_dtype
+                )
+                audio_tower.init_latency_mask(
+                    config.audio_latency_block_size, dtype=config.torch_dtype
+                )
+            else:
+                assert config.audio_latency_block_size in (
+                    None,
+                    0,
+                ), "only whisper audio tower supports audio latency masking, got non-zero value for 'audio_latency_block_size'"
+                audio_tower = transformers.AutoModel.from_pretrained(
+                    config.audio_model_id, torch_dtype=config.torch_dtype
+                )
+        else:
+            if "whisper" in config.audio_config._name_or_path.lower():
+                audio_tower = ModifiedWhisperEncoder(config.audio_config)
+                audio_tower.init_latency_mask(
+                    config.audio_latency_block_size, dtype=config.torch_dtype
+                )
+            else:
+                assert config.audio_latency_block_size in (
+                    None,
+                    0,
+                ), "only whisper audio tower supports audio latency masking, got non-zero value for 'audio_latency_block_size'"
+                with transformers.modeling_utils.no_init_weights():
+                    # we only ever use from_config if the weights are retrained, hence initializing is not
+                    # required. This makes the model quite creation faster since init on CPU is quite slow.
+                    audio_tower = transformers.AutoModel.from_config(
+                        config.audio_config
+                    )
+
+        if isinstance(
+            audio_tower,
+            (transformers.Wav2Vec2BertModel, transformers.WhisperModel),
+        ):
+            # For these models we only need the encoder part
+            # Wav2Vec2BertModel -> Wav2Vec2BertEncoder
+            # WhisperModel -> WhisperEncoder
+            audio_tower = audio_tower.encoder
+
+        audio_tower = apply_lora(audio_tower, config.audio_model_lora_config)
+        return audio_tower
+
+    @classmethod
+    def _create_language_model(
+        cls, config: UltravoxConfig
+    ) -> transformers.LlamaForCausalLM:
+        if config.text_model_id is not None:
+            language_model = transformers.AutoModelForCausalLM.from_pretrained(
+                config.text_model_id,
+                attn_implementation=config._attn_implementation,
+                torch_dtype=config.torch_dtype,
+            )
+        else:
+            with transformers.modeling_utils.no_init_weights():
+                # we only ever use from_config if the weights are retrained, hence initializing is not
+                # required. This makes the model quite creation faster since init on CPU is quite slow.
+                language_model = transformers.AutoModelForCausalLM.from_config(
+                    config.text_config,
+                    attn_implementation=config._attn_implementation,
+                    torch_dtype=config.torch_dtype,
+                )
+
+        language_model = apply_lora(language_model, config.text_model_lora_config)
+        return language_model
+
+    def merge_and_unload(self):
+        if isinstance(self.language_model, peft.PeftModel):
+            self.language_model = self.language_model.merge_and_unload()
+            # no need to download base language model weights anymore, so we can remove the id
+            self.config.text_model_id = None
+            self.keep_params.update(
+                set(
+                    [
+                        f"language_model.{name}"
+                        for name, _ in self.language_model.named_parameters()
+                    ]
+                )
+            )
+
+        if isinstance(self.audio_tower, peft.PeftModel):
+            self.audio_tower = self.audio_tower.merge_and_unload()
+            # no need to download base audio model weights anymore, so we can remove the id
+            self.config.audio_model_id = None
+            self.keep_params.update(
+                set(
+                    [
+                        f"audio_tower.{name}"
+                        for name, _ in self.audio_tower.named_parameters()
+                    ]
+                )
+            )
+
+        for param in ["text_model_lora_config", "audio_model_lora_config"]:
+            if hasattr(self.config, param):
+                delattr(self.config, param)
+
+    def push_to_hub(self, *args, **kwargs):
+        self.merge_and_unload()
+        return super().push_to_hub(*args, **kwargs)
+
+    def diff_state_dict(
+        self, state_dict: Optional[Dict[str, Any]] = None
+    ) -> Dict[str, Any]:
+        if state_dict is None:
+            state_dict = super().state_dict()
+
+        named_params = dict(self.named_parameters())
+
+        state_dict = {
+            k: v
+            for k, v in state_dict.items()
+            if k in self.keep_params
+            or (k in named_params and named_params[k].requires_grad)
+        }
+
+        return state_dict
+
+    def save_pretrained(
+        self, *args, state_dict: Optional[Dict[str, Any]] = None, **kwargs
+    ):
+        state_dict = self.diff_state_dict(state_dict)
+
+        super().save_pretrained(*args, state_dict=state_dict, **kwargs)
+
+    def _pre_load_state_dict_hook(self, state_dict: Dict[str, Any], *args, **kwargs):
+        self.keep_params.update(set(state_dict.keys()))
+
+    def print_trainable_parameters(self):
+        """
+        Prints the number of trainable parameters in the model (reuses Peft model's method)
+        """
+        count_params = peft.peft_model.PeftModel.get_nb_trainable_parameters
+
+        trainable_params, all_param = count_params(self)
+
+        logging.info(
+            f"trainable params: {trainable_params:,d} || all params: {all_param:,d}"
+            f" || trainable%: {100 * trainable_params / all_param:.1f}%"
+        )
+
+        lm_trainable_params, lm_all_params = count_params(self.language_model)
+        audio_trainable_params, audio_all_params = count_params(self.audio_tower)
+
+        projector_trainable_params = (
+            trainable_params - lm_trainable_params - audio_trainable_params
+        )
+        projector_all_params = all_param - lm_all_params - audio_all_params
+
+        logging.info(
+            f"Trainable%:   "
+            f" LLM: {100 * lm_trainable_params / lm_all_params:.1f}%"
+            f" || Audio Encoder: {100 * audio_trainable_params / audio_all_params:.1f}%"
+            f" || Projector: {100 * projector_trainable_params / projector_all_params:.1f}%"
+        )
+
+
+# TODO: refactor common parts to a shared module
+def is_cache_empty(
+    past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]]
+) -> bool:
+    """
+    Check if the cache is empty.
+    """
+    if past_key_values is None:
+        return True
+    if isinstance(past_key_values, tuple):
+        return all(len(c) == 0 for c in past_key_values)
+    return past_key_values.get_seq_length() == 0
+
+
+def apply_lora(model: torch.nn.Module, lora_config: dict) -> torch.nn.Module:
+    """
+    Applies LoRA finetuning to the model. If the `r` parameter is set to 0, the model is frozen instead.
+    """
+    unfreeze_layers = lora_config.pop("unfreeze_layers", None)
+    lora_config = peft.LoraConfig(**lora_config or {})
+
+    if lora_config.r == 0:
+        # freeze the model entirely, except for the specified layers
+        for name, param in model.named_parameters():
+            if not unfreeze_layers or not any(
+                re.match(layer, name) for layer in unfreeze_layers
+            ):
+                param.requires_grad = False
+            else:
+                logging.info(f"Unfreezing layer: {name} with #{param.numel()} params")
+    else:
+        model = peft.get_peft_model(model, lora_config)
+
+    return model
+
+
+class StackAudioFrames(nn.Module):
+    """
+    Stack the audio embedding frames to reduce the sequence length by a factor of `stack_factor`.
+
+    The number of output frames will be `ceil(T / stack_factor) + 1` where `T` is the number of input frames.
+    NOTE: the extra +1 is intentional: in case the number of audio tokens are over-estimated by the processor,
+    we want to make sure `processor.audio_token_replacement` (i.e. EOS) doesn't get leaked into the middle of embeddings.
+    In most cases this extra padding will get removed in the model's forward function so it has no effect.
+    """
+
+    def __init__(self, stack_factor: int = 8):
+        super().__init__()
+        self.stack_factor = stack_factor
+
+    def forward(self, audio_embeds: torch.Tensor) -> torch.Tensor:
+        B, T, C = audio_embeds.shape
+        T_pad = (T + self.stack_factor - 1) // self.stack_factor * self.stack_factor
+        audio_embeds = F.pad(audio_embeds, (0, 0, 0, T_pad - T + self.stack_factor))
+        B, T, C = audio_embeds.shape
+        audio_embeds = audio_embeds.view(
+            B, T // self.stack_factor, C * self.stack_factor
+        )
+        return audio_embeds
+
+
+class RMSNorm(transformers.models.llama.modeling_llama.LlamaRMSNorm):
+    def __init__(self, hidden_size: int, init: float = 1, eps: float = 1e-6):
+        super().__init__(hidden_size=hidden_size, eps=eps)
+        self.weight.data.fill_(init)
+
+
+class SwiGLU(nn.Module):
+    def forward(self, x):
+        x, gate = x.chunk(2, dim=-1)
+        return F.silu(gate) * x
+
+
+class UltravoxProjector(nn.Module):
+    def __init__(self, config: UltravoxConfig):
+        super().__init__()
+        self.hidden_dim = config.hidden_size
+        self._pad_and_stack = StackAudioFrames(config.stack_factor)
+        dim_in = config.audio_config.hidden_size * config.stack_factor
+        self.ln_pre = RMSNorm(dim_in, init=config.norm_init)
+        self.linear_1 = nn.Linear(dim_in, self.hidden_dim, bias=False)
+        dim_mid = self.hidden_dim
+        self.act = transformers.activations.get_activation(config.projector_act)
+        dim_mid = dim_mid // 2 if config.projector_act == "swiglu" else dim_mid
+        dim_out = config.text_config.hidden_size
+        self.linear_2 = nn.Linear(dim_mid, dim_out, bias=False)
+
+        # Ultravox v0.4.1 and below uses layer_norm after the second linear layer,
+        # while v0.5.0 and above uses layer_norm after the first linear layer.
+        if config.projector_ln_mid:
+            self.ln_mid: nn.Module = RMSNorm(dim_mid, init=config.norm_init)
+            self.ln_post: nn.Module = nn.Identity()
+        else:
+            self.ln_mid = nn.Identity()
+            self.ln_post = RMSNorm(dim_out, init=config.norm_init)
+
+    def forward(self, audio_features: torch.Tensor) -> torch.Tensor:
+        audio_features = self._pad_and_stack(audio_features)
+        audio_features = self.ln_pre(audio_features)
+        hidden_states = self.linear_1(audio_features)
+        hidden_states = self.act(hidden_states)
+        hidden_states = self.ln_mid(hidden_states)
+        hidden_states = self.linear_2(hidden_states)
+        hidden_states = self.ln_post(hidden_states)
+        return hidden_states
+
+
+class ModifiedWhisperEncoder(
+    whisper.WhisperEncoder, transformers.modeling_utils.ModuleUtilsMixin
+):
+    """
+    Encoder portion of OpenAI's Whisper model.
+
+    This implementation is a slightly modified version of HF Transformers' Whisper Encoder, with only a few fixes:
+    1. base_model_prefix updated to allow for doing `.from_pretrained` directly on the encoder
+    2. allow less than 30 second of audio padding to be passed in:
+        - relaxed ValueError check for `input_features` length to be less than or equal to `expected_seq_length` instead of strictly equal
+        - embed_pos is now sliced to match the length of `inputs_embeds`
+
+    Original: https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/modeling_whisper.py
+    """
+
+    base_model_prefix = "model.encoder"
+    _no_split_modules = ["WhisperEncoderLayer"]
+
+    def __init__(self, config: transformers.WhisperConfig):
+        super().__init__(config)
+        self.config.is_decoder = False
+
+    def init_latency_mask(self, audio_latency_block_size: int, dtype: torch.dtype):
+        if audio_latency_block_size is None:
+            self.audio_streaming_mask = None
+            return
+
+        # maximum sequence length
+        max_seqlen = (
+            self.config.max_source_positions
+            * self.conv1.stride[0]
+            * self.conv2.stride[0]
+        )
+        assert (
+            max_seqlen > 0
+        ), f"maximum sequence length must be positive, got {max_seqlen}"
+        assert (
+            max_seqlen % audio_latency_block_size == 0
+        ), f"audio_latency_block_size {audio_latency_block_size} must divide {max_seqlen} evenly."
+        # Given the block size, we calculate number of blocks.
+        audio_latency_nblocks = max_seqlen // audio_latency_block_size
+        audio_streaming_mask = (
+            torch.tril(
+                torch.ones(audio_latency_nblocks, audio_latency_nblocks),
+                diagonal=0,
+            )
+            .repeat_interleave(audio_latency_block_size, dim=0)
+            .repeat_interleave(audio_latency_block_size, dim=1)
+        )
+        audio_streaming_mask = (1.0 - audio_streaming_mask) * torch.finfo(dtype).min
+        audio_streaming_mask = audio_streaming_mask[None, None, :, :]
+        self.register_buffer(
+            "audio_streaming_mask", audio_streaming_mask, persistent=False
+        )
+
+    def forward(
+        self,
+        input_features,
+        audio_len=None,
+        head_mask=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+    ):
+        expected_seq_length = (
+            self.config.max_source_positions
+            * self.conv1.stride[0]
+            * self.conv2.stride[0]
+        )
+        if input_features.shape[-1] > expected_seq_length:
+            raise ValueError(
+                f"Whisper expects the mel input features to be of length {expected_seq_length} or less, but found {input_features.shape[-1]}. Make sure to pad the input mel features to {expected_seq_length}."
+            )
+
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+        inputs_embeds = nn.functional.gelu(self.conv1(input_features))
+        inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds))
+
+        inputs_embeds = inputs_embeds.permute(0, 2, 1)
+        embed_pos = self.embed_positions.weight[: inputs_embeds.size(-2)]
+
+        hidden_states = inputs_embeds + embed_pos
+        hidden_states = nn.functional.dropout(
+            hidden_states, p=self.dropout, training=self.training
+        )
+
+        encoder_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+
+        # Create attention mask based on audio lengths to mask out padding tokens
+        # For each sample in batch:
+        # - Convert raw audio length to feature length after convolutions
+        # - Create boolean mask that is True for valid positions and False for padding
+        # - Convert to extended attention mask format expected by transformer layers
+        #   (1.0 for positions to attend to, large negative for positions to ignore)
+        # This masking ensures consistent behavior between training and inference
+        # by preventing the model from attending to padding tokens in both cases
+        attention_mask = None
+        if audio_len != None:
+            audio_feature_len = self._get_feat_extract_output_lengths(audio_len)
+            max_seq_len = hidden_states.shape[1]
+            attention_mask = torch.arange(max_seq_len, device=hidden_states.device)[
+                None, :
+            ].lt(audio_feature_len.view(-1, 1))
+            attention_mask = self.get_extended_attention_mask(
+                attention_mask,
+                None,
+                device=hidden_states.device,
+                dtype=hidden_states.dtype,
+            )
+
+        if self.audio_streaming_mask is not None:
+            seqlen = hidden_states.size(-2)
+            if attention_mask is not None:
+                attention_mask = torch.minimum(
+                    self.audio_streaming_mask[:, :, :seqlen, :seqlen], attention_mask
+                )  # merge
+            else:
+                attention_mask = self.audio_streaming_mask[:, :, :seqlen, :seqlen]
+            attention_mask = attention_mask.to(hidden_states.dtype)
+
+        # check if head_mask has a correct number of layers specified if desired
+        if head_mask is not None:
+            assert head_mask.size()[0] == (
+                len(self.layers)
+            ), f"The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.size()[0]}."
+
+        for idx, encoder_layer in enumerate(self.layers):
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
+            to_drop = False
+            if self.training:
+                dropout_probability = torch.rand([])
+                if dropout_probability < self.layerdrop:  # skip the layer
+                    to_drop = True
+
+            if to_drop:
+                layer_outputs = (None, None)
+            else:
+                if self.gradient_checkpointing and self.training:
+                    layer_outputs = self._gradient_checkpointing_func(
+                        encoder_layer.__call__,
+                        hidden_states,
+                        attention_mask,
+                        (head_mask[idx] if head_mask is not None else None),
+                        output_attentions,
+                    )
+                else:
+                    layer_outputs = encoder_layer(
+                        hidden_states,
+                        attention_mask,
+                        layer_head_mask=(
+                            head_mask[idx] if head_mask is not None else None
+                        ),
+                        output_attentions=output_attentions,
+                    )
+
+                hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+        hidden_states = self.layer_norm(hidden_states)
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [hidden_states, encoder_states, all_attentions]
+                if v is not None
+            )
+        return transformers.modeling_outputs.BaseModelOutput(
+            last_hidden_state=hidden_states,
+            hidden_states=encoder_states,
+            attentions=all_attentions,
+        )
+
+
+UltravoxConfig.register_for_auto_class()
+UltravoxModel.register_for_auto_class()
+
+transformers.AutoConfig.register("ultravox", UltravoxConfig)
+transformers.AutoModel.register(UltravoxConfig, UltravoxModel)
+
+transformers.activations.ACT2FN["swiglu"] = SwiGLU

checkpoint-14400/config.json

@@ -0,0 +1,44 @@
+{
+  "architectures": [
+    "UltravoxModel"
+  ],
+  "audio_latency_block_size": null,
+  "audio_model_id": "openai/whisper-large-v3-turbo",
+  "audio_model_lora_config": {
+    "lora_alpha": 8,
+    "r": 0,
+    "target_modules": [
+      "k_proj",
+      "q_proj",
+      "linear_k",
+      "linear_q"
+    ]
+  },
+  "auto_map": {
+    "AutoConfig": "ultravox_config.UltravoxConfig",
+    "AutoModel": "ultravox_model.UltravoxModel"
+  },
+  "hidden_size": 4096,
+  "ignore_index": -100,
+  "initializer_range": 0.02,
+  "model_type": "ultravox",
+  "norm_init": 0.4,
+  "pad_token_id": 2,
+  "projector_act": "swiglu",
+  "projector_ln_mid": false,
+  "stack_factor": 8,
+  "text_model_id": "HuggingFaceTB/SmolLM2-1.7B-Instruct",
+  "text_model_lora_config": {
+    "lora_alpha": 8,
+    "r": 0,
+    "target_modules": [
+      "k_proj",
+      "q_proj",
+      "linear_k",
+      "linear_q"
+    ]
+  },
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.47.0",
+  "vocab_size": 49152
+}

checkpoint-14400/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 2,
+  "transformers_version": "4.47.0"
+}

checkpoint-14400/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8bdd0552282a75251fa0430873e29cc491e32579e4b6657f51579c7f5aa99ef
+size 92299736

checkpoint-14400/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18631729e7d0e9597a4c02ca2b6facf9c52b83c22ea7c0e485c889fcb50a476f
+size 184602962

checkpoint-14400/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:178471e10e4ada9e88a29563afc3ad35ab4ec31626343f0d29085b1cd7647911
+size 14244

checkpoint-14400/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c57798d3fa88e9f7f19098498af2fbd85578814b976457ff05f5a72957807e9
+size 1064

checkpoint-14400/special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|im_end|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-14400/tokenizer_config.json

@@ -0,0 +1,155 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|im_end|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

checkpoint-14400/trainer_state.json

@@ -0,0 +1,1174 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.11984619738002897,
+  "eval_steps": 1000,
+  "global_step": 14400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 8.322652595835345e-06,
+      "grad_norm": 14.9375,
+      "learning_rate": 2e-06,
+      "loss": 0.6484,
+      "step": 1
+    },
+    {
+      "epoch": 0.0008322652595835345,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.0002,
+      "loss": 0.4345,
+      "step": 100
+    },
+    {
+      "epoch": 0.001664530519167069,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 0.0004,
+      "loss": 0.2537,
+      "step": 200
+    },
+    {
+      "epoch": 0.0024967957787506035,
+      "grad_norm": 0.55859375,
+      "learning_rate": 0.0006,
+      "loss": 0.2344,
+      "step": 300
+    },
+    {
+      "epoch": 0.003329061038334138,
+      "grad_norm": 0.154296875,
+      "learning_rate": 0.0008,
+      "loss": 0.222,
+      "step": 400
+    },
+    {
+      "epoch": 0.004161326297917672,
+      "grad_norm": 0.126953125,
+      "learning_rate": 0.001,
+      "loss": 0.2142,
+      "step": 500
+    },
+    {
+      "epoch": 0.004993591557501207,
+      "grad_norm": 0.12353515625,
+      "learning_rate": 0.0012,
+      "loss": 0.2011,
+      "step": 600
+    },
+    {
+      "epoch": 0.005825856817084741,
+      "grad_norm": 0.10693359375,
+      "learning_rate": 0.0014,
+      "loss": 0.1897,
+      "step": 700
+    },
+    {
+      "epoch": 0.006658122076668276,
+      "grad_norm": 0.10595703125,
+      "learning_rate": 0.0016,
+      "loss": 0.1775,
+      "step": 800
+    },
+    {
+      "epoch": 0.0074903873362518105,
+      "grad_norm": 0.0859375,
+      "learning_rate": 0.0018000000000000002,
+      "loss": 0.1615,
+      "step": 900
+    },
+    {
+      "epoch": 0.008322652595835344,
+      "grad_norm": 0.083984375,
+      "learning_rate": 0.002,
+      "loss": 0.1444,
+      "step": 1000
+    },
+    {
+      "epoch": 0.008322652595835344,
+      "eval_peoplespeech-clean-transcription_loss": 2.2110702991485596,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.5405,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.708,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.839,
+      "step": 1000
+    },
+    {
+      "epoch": 0.009154917855418878,
+      "grad_norm": 0.0771484375,
+      "learning_rate": 0.001999725185109816,
+      "loss": 0.1323,
+      "step": 1100
+    },
+    {
+      "epoch": 0.009987183115002414,
+      "grad_norm": 0.083984375,
+      "learning_rate": 0.0019989008914857113,
+      "loss": 0.1242,
+      "step": 1200
+    },
+    {
+      "epoch": 0.010819448374585948,
+      "grad_norm": 0.07861328125,
+      "learning_rate": 0.00199752757218401,
+      "loss": 0.1162,
+      "step": 1300
+    },
+    {
+      "epoch": 0.011651713634169482,
+      "grad_norm": 0.07275390625,
+      "learning_rate": 0.001995605982021898,
+      "loss": 0.1128,
+      "step": 1400
+    },
+    {
+      "epoch": 0.012483978893753018,
+      "grad_norm": 0.07666015625,
+      "learning_rate": 0.0019931371771625545,
+      "loss": 0.1094,
+      "step": 1500
+    },
+    {
+      "epoch": 0.013316244153336551,
+      "grad_norm": 0.06201171875,
+      "learning_rate": 0.001990122514534651,
+      "loss": 0.1052,
+      "step": 1600
+    },
+    {
+      "epoch": 0.014148509412920085,
+      "grad_norm": 0.058837890625,
+      "learning_rate": 0.0019865636510865464,
+      "loss": 0.1022,
+      "step": 1700
+    },
+    {
+      "epoch": 0.014980774672503621,
+      "grad_norm": 0.0625,
+      "learning_rate": 0.001982462542875576,
+      "loss": 0.1011,
+      "step": 1800
+    },
+    {
+      "epoch": 0.015813039932087155,
+      "grad_norm": 0.06787109375,
+      "learning_rate": 0.001977821443992945,
+      "loss": 0.0983,
+      "step": 1900
+    },
+    {
+      "epoch": 0.01664530519167069,
+      "grad_norm": 0.0537109375,
+      "learning_rate": 0.001972642905324813,
+      "loss": 0.0975,
+      "step": 2000
+    },
+    {
+      "epoch": 0.01664530519167069,
+      "eval_peoplespeech-clean-transcription_loss": 1.6914767026901245,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.5301,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.716,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.839,
+      "step": 2000
+    },
+    {
+      "epoch": 0.017477570451254223,
+      "grad_norm": 0.05859375,
+      "learning_rate": 0.0019669297731502505,
+      "loss": 0.0947,
+      "step": 2100
+    },
+    {
+      "epoch": 0.018309835710837757,
+      "grad_norm": 0.062255859375,
+      "learning_rate": 0.00196068518757684,
+      "loss": 0.0935,
+      "step": 2200
+    },
+    {
+      "epoch": 0.019142100970421294,
+      "grad_norm": 0.059326171875,
+      "learning_rate": 0.001953912580814779,
+      "loss": 0.0911,
+      "step": 2300
+    },
+    {
+      "epoch": 0.019974366230004828,
+      "grad_norm": 0.060302734375,
+      "learning_rate": 0.0019466156752904343,
+      "loss": 0.0904,
+      "step": 2400
+    },
+    {
+      "epoch": 0.020806631489588362,
+      "grad_norm": 0.0615234375,
+      "learning_rate": 0.0019387984816003866,
+      "loss": 0.0882,
+      "step": 2500
+    },
+    {
+      "epoch": 0.021638896749171896,
+      "grad_norm": 0.056884765625,
+      "learning_rate": 0.0019304652963070869,
+      "loss": 0.0874,
+      "step": 2600
+    },
+    {
+      "epoch": 0.02247116200875543,
+      "grad_norm": 0.049072265625,
+      "learning_rate": 0.0019216206995773372,
+      "loss": 0.0871,
+      "step": 2700
+    },
+    {
+      "epoch": 0.023303427268338964,
+      "grad_norm": 0.053955078125,
+      "learning_rate": 0.0019122695526648968,
+      "loss": 0.0856,
+      "step": 2800
+    },
+    {
+      "epoch": 0.0241356925279225,
+      "grad_norm": 0.05517578125,
+      "learning_rate": 0.0019024169952385887,
+      "loss": 0.0845,
+      "step": 2900
+    },
+    {
+      "epoch": 0.024967957787506035,
+      "grad_norm": 0.0546875,
+      "learning_rate": 0.0018920684425573864,
+      "loss": 0.0852,
+      "step": 3000
+    },
+    {
+      "epoch": 0.024967957787506035,
+      "eval_peoplespeech-clean-transcription_loss": 1.6414048671722412,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.8151,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.521,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.815,
+      "step": 3000
+    },
+    {
+      "epoch": 0.02580022304708957,
+      "grad_norm": 0.052978515625,
+      "learning_rate": 0.0018812295824940284,
+      "loss": 0.0836,
+      "step": 3100
+    },
+    {
+      "epoch": 0.026632488306673103,
+      "grad_norm": 0.05126953125,
+      "learning_rate": 0.0018699063724087904,
+      "loss": 0.0821,
+      "step": 3200
+    },
+    {
+      "epoch": 0.027464753566256637,
+      "grad_norm": 0.056884765625,
+      "learning_rate": 0.0018581050358751443,
+      "loss": 0.0816,
+      "step": 3300
+    },
+    {
+      "epoch": 0.02829701882584017,
+      "grad_norm": 0.045654296875,
+      "learning_rate": 0.0018458320592590974,
+      "loss": 0.0814,
+      "step": 3400
+    },
+    {
+      "epoch": 0.029129284085423705,
+      "grad_norm": 0.04638671875,
+      "learning_rate": 0.0018330941881540914,
+      "loss": 0.0791,
+      "step": 3500
+    },
+    {
+      "epoch": 0.029961549345007242,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 0.0018198984236734246,
+      "loss": 0.0804,
+      "step": 3600
+    },
+    {
+      "epoch": 0.030793814604590776,
+      "grad_norm": 0.054931640625,
+      "learning_rate": 0.0018062520186022297,
+      "loss": 0.0802,
+      "step": 3700
+    },
+    {
+      "epoch": 0.03162607986417431,
+      "grad_norm": 0.054931640625,
+      "learning_rate": 0.0017921624734111292,
+      "loss": 0.0805,
+      "step": 3800
+    },
+    {
+      "epoch": 0.032458345123757844,
+      "grad_norm": 0.048095703125,
+      "learning_rate": 0.001777637532133752,
+      "loss": 0.079,
+      "step": 3900
+    },
+    {
+      "epoch": 0.03329061038334138,
+      "grad_norm": 0.0458984375,
+      "learning_rate": 0.0017626851781103819,
+      "loss": 0.0759,
+      "step": 4000
+    },
+    {
+      "epoch": 0.03329061038334138,
+      "eval_peoplespeech-clean-transcription_loss": 1.5952188968658447,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.4647,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.762,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.845,
+      "step": 4000
+    },
+    {
+      "epoch": 0.03412287564292491,
+      "grad_norm": 0.06494140625,
+      "learning_rate": 0.001747313629600077,
+      "loss": 0.0774,
+      "step": 4100
+    },
+    {
+      "epoch": 0.034955140902508446,
+      "grad_norm": 0.043701171875,
+      "learning_rate": 0.001731531335263669,
+      "loss": 0.0774,
+      "step": 4200
+    },
+    {
+      "epoch": 0.03578740616209198,
+      "grad_norm": 0.048095703125,
+      "learning_rate": 0.0017153469695201276,
+      "loss": 0.0769,
+      "step": 4300
+    },
+    {
+      "epoch": 0.03661967142167551,
+      "grad_norm": 0.051513671875,
+      "learning_rate": 0.0016987694277788418,
+      "loss": 0.0763,
+      "step": 4400
+    },
+    {
+      "epoch": 0.037451936681259054,
+      "grad_norm": 0.047607421875,
+      "learning_rate": 0.001681807821550438,
+      "loss": 0.0774,
+      "step": 4500
+    },
+    {
+      "epoch": 0.03828420194084259,
+      "grad_norm": 0.040283203125,
+      "learning_rate": 0.0016644714734388218,
+      "loss": 0.0748,
+      "step": 4600
+    },
+    {
+      "epoch": 0.03911646720042612,
+      "grad_norm": 0.05419921875,
+      "learning_rate": 0.0016467699120171987,
+      "loss": 0.0755,
+      "step": 4700
+    },
+    {
+      "epoch": 0.039948732460009656,
+      "grad_norm": 0.048828125,
+      "learning_rate": 0.001628712866590885,
+      "loss": 0.0732,
+      "step": 4800
+    },
+    {
+      "epoch": 0.04078099771959319,
+      "grad_norm": 0.046875,
+      "learning_rate": 0.0016103102618497923,
+      "loss": 0.0757,
+      "step": 4900
+    },
+    {
+      "epoch": 0.041613262979176724,
+      "grad_norm": 0.042236328125,
+      "learning_rate": 0.0015915722124135226,
+      "loss": 0.0738,
+      "step": 5000
+    },
+    {
+      "epoch": 0.041613262979176724,
+      "eval_peoplespeech-clean-transcription_loss": 1.5546748638153076,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.345,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.849,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.856,
+      "step": 5000
+    },
+    {
+      "epoch": 0.04244552823876026,
+      "grad_norm": 0.045166015625,
+      "learning_rate": 0.001572509017272072,
+      "loss": 0.0742,
+      "step": 5100
+    },
+    {
+      "epoch": 0.04327779349834379,
+      "grad_norm": 0.0556640625,
+      "learning_rate": 0.0015531311541251993,
+      "loss": 0.0746,
+      "step": 5200
+    },
+    {
+      "epoch": 0.044110058757927326,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 0.0015334492736235703,
+      "loss": 0.073,
+      "step": 5300
+    },
+    {
+      "epoch": 0.04494232401751086,
+      "grad_norm": 0.044921875,
+      "learning_rate": 0.0015134741935148419,
+      "loss": 0.0727,
+      "step": 5400
+    },
+    {
+      "epoch": 0.045774589277094394,
+      "grad_norm": 0.05029296875,
+      "learning_rate": 0.0014932168926979072,
+      "loss": 0.0726,
+      "step": 5500
+    },
+    {
+      "epoch": 0.04660685453667793,
+      "grad_norm": 0.048095703125,
+      "learning_rate": 0.0014726885051885652,
+      "loss": 0.0718,
+      "step": 5600
+    },
+    {
+      "epoch": 0.04743911979626146,
+      "grad_norm": 0.046875,
+      "learning_rate": 0.0014519003139999338,
+      "loss": 0.0736,
+      "step": 5700
+    },
+    {
+      "epoch": 0.048271385055845,
+      "grad_norm": 0.051513671875,
+      "learning_rate": 0.0014308637449409706,
+      "loss": 0.0724,
+      "step": 5800
+    },
+    {
+      "epoch": 0.049103650315428536,
+      "grad_norm": 0.053955078125,
+      "learning_rate": 0.0014095903603365066,
+      "loss": 0.0724,
+      "step": 5900
+    },
+    {
+      "epoch": 0.04993591557501207,
+      "grad_norm": 0.037841796875,
+      "learning_rate": 0.0013880918526722496,
+      "loss": 0.071,
+      "step": 6000
+    },
+    {
+      "epoch": 0.04993591557501207,
+      "eval_peoplespeech-clean-transcription_loss": 1.5297455787658691,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.3364,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.855,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.857,
+      "step": 6000
+    },
+    {
+      "epoch": 0.050768180834595604,
+      "grad_norm": 0.05029296875,
+      "learning_rate": 0.0013663800381682463,
+      "loss": 0.0713,
+      "step": 6100
+    },
+    {
+      "epoch": 0.05160044609417914,
+      "grad_norm": 0.037109375,
+      "learning_rate": 0.0013444668502843329,
+      "loss": 0.0701,
+      "step": 6200
+    },
+    {
+      "epoch": 0.05243271135376267,
+      "grad_norm": 0.03564453125,
+      "learning_rate": 0.0013223643331611537,
+      "loss": 0.0708,
+      "step": 6300
+    },
+    {
+      "epoch": 0.053264976613346206,
+      "grad_norm": 0.04443359375,
+      "learning_rate": 0.001300084635000341,
+      "loss": 0.0707,
+      "step": 6400
+    },
+    {
+      "epoch": 0.05409724187292974,
+      "grad_norm": 0.033935546875,
+      "learning_rate": 0.0012776400013875004,
+      "loss": 0.0704,
+      "step": 6500
+    },
+    {
+      "epoch": 0.054929507132513274,
+      "grad_norm": 0.046142578125,
+      "learning_rate": 0.0012550427685616766,
+      "loss": 0.0707,
+      "step": 6600
+    },
+    {
+      "epoch": 0.05576177239209681,
+      "grad_norm": 0.04150390625,
+      "learning_rate": 0.0012323053566349834,
+      "loss": 0.0696,
+      "step": 6700
+    },
+    {
+      "epoch": 0.05659403765168034,
+      "grad_norm": 0.042236328125,
+      "learning_rate": 0.0012094402627661448,
+      "loss": 0.0705,
+      "step": 6800
+    },
+    {
+      "epoch": 0.057426302911263875,
+      "grad_norm": 0.035888671875,
+      "learning_rate": 0.0011864600542916813,
+      "loss": 0.0686,
+      "step": 6900
+    },
+    {
+      "epoch": 0.05825856817084741,
+      "grad_norm": 0.0498046875,
+      "learning_rate": 0.0011633773618185302,
+      "loss": 0.0687,
+      "step": 7000
+    },
+    {
+      "epoch": 0.05825856817084741,
+      "eval_peoplespeech-clean-transcription_loss": 1.518404483795166,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.4703,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.758,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.845,
+      "step": 7000
+    },
+    {
+      "epoch": 0.05909083343043095,
+      "grad_norm": 0.034423828125,
+      "learning_rate": 0.0011402048722818862,
+      "loss": 0.0693,
+      "step": 7100
+    },
+    {
+      "epoch": 0.059923098690014484,
+      "grad_norm": 0.047119140625,
+      "learning_rate": 0.0011169553219720827,
+      "loss": 0.0697,
+      "step": 7200
+    },
+    {
+      "epoch": 0.06075536394959802,
+      "grad_norm": 0.0576171875,
+      "learning_rate": 0.001093641489534351,
+      "loss": 0.0685,
+      "step": 7300
+    },
+    {
+      "epoch": 0.06158762920918155,
+      "grad_norm": 0.052490234375,
+      "learning_rate": 0.001070276188945293,
+      "loss": 0.0666,
+      "step": 7400
+    },
+    {
+      "epoch": 0.062419894468765086,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 0.00104687226246994,
+      "loss": 0.0692,
+      "step": 7500
+    },
+    {
+      "epoch": 0.06325215972834862,
+      "grad_norm": 0.04052734375,
+      "learning_rate": 0.0010234425736032607,
+      "loss": 0.069,
+      "step": 7600
+    },
+    {
+      "epoch": 0.06408442498793215,
+      "grad_norm": 0.06298828125,
+      "learning_rate": 0.001,
+      "loss": 0.0682,
+      "step": 7700
+    },
+    {
+      "epoch": 0.06491669024751569,
+      "grad_norm": 0.04736328125,
+      "learning_rate": 0.0009765574263967396,
+      "loss": 0.0676,
+      "step": 7800
+    },
+    {
+      "epoch": 0.06574895550709922,
+      "grad_norm": 0.064453125,
+      "learning_rate": 0.0009531277375300599,
+      "loss": 0.0685,
+      "step": 7900
+    },
+    {
+      "epoch": 0.06658122076668276,
+      "grad_norm": 0.037109375,
+      "learning_rate": 0.0009297238110547074,
+      "loss": 0.0677,
+      "step": 8000
+    },
+    {
+      "epoch": 0.06658122076668276,
+      "eval_peoplespeech-clean-transcription_loss": 1.5178765058517456,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.3361,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.855,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.857,
+      "step": 8000
+    },
+    {
+      "epoch": 0.06741348602626629,
+      "grad_norm": 0.041259765625,
+      "learning_rate": 0.0009063585104656494,
+      "loss": 0.068,
+      "step": 8100
+    },
+    {
+      "epoch": 0.06824575128584982,
+      "grad_norm": 0.0390625,
+      "learning_rate": 0.0008830446780279176,
+      "loss": 0.0681,
+      "step": 8200
+    },
+    {
+      "epoch": 0.06907801654543336,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 0.0008597951277181142,
+      "loss": 0.0677,
+      "step": 8300
+    },
+    {
+      "epoch": 0.06991028180501689,
+      "grad_norm": 0.054443359375,
+      "learning_rate": 0.0008366226381814697,
+      "loss": 0.0679,
+      "step": 8400
+    },
+    {
+      "epoch": 0.07074254706460043,
+      "grad_norm": 0.036865234375,
+      "learning_rate": 0.000813539945708319,
+      "loss": 0.0685,
+      "step": 8500
+    },
+    {
+      "epoch": 0.07157481232418396,
+      "grad_norm": 0.036865234375,
+      "learning_rate": 0.0007905597372338558,
+      "loss": 0.0678,
+      "step": 8600
+    },
+    {
+      "epoch": 0.07240707758376749,
+      "grad_norm": 0.03466796875,
+      "learning_rate": 0.0007676946433650169,
+      "loss": 0.0671,
+      "step": 8700
+    },
+    {
+      "epoch": 0.07323934284335103,
+      "grad_norm": 0.03857421875,
+      "learning_rate": 0.0007449572314383236,
+      "loss": 0.0659,
+      "step": 8800
+    },
+    {
+      "epoch": 0.07407160810293456,
+      "grad_norm": 0.04248046875,
+      "learning_rate": 0.0007223599986124993,
+      "loss": 0.0657,
+      "step": 8900
+    },
+    {
+      "epoch": 0.07490387336251811,
+      "grad_norm": 0.0517578125,
+      "learning_rate": 0.0006999153649996595,
+      "loss": 0.0662,
+      "step": 9000
+    },
+    {
+      "epoch": 0.07490387336251811,
+      "eval_peoplespeech-clean-transcription_loss": 1.49990975856781,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.6057,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.663,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.833,
+      "step": 9000
+    },
+    {
+      "epoch": 0.07573613862210164,
+      "grad_norm": 0.05419921875,
+      "learning_rate": 0.0006776356668388464,
+      "loss": 0.0659,
+      "step": 9100
+    },
+    {
+      "epoch": 0.07656840388168518,
+      "grad_norm": 0.041015625,
+      "learning_rate": 0.0006555331497156671,
+      "loss": 0.0656,
+      "step": 9200
+    },
+    {
+      "epoch": 0.07740066914126871,
+      "grad_norm": 0.08203125,
+      "learning_rate": 0.0006336199618317538,
+      "loss": 0.0663,
+      "step": 9300
+    },
+    {
+      "epoch": 0.07823293440085224,
+      "grad_norm": 0.03271484375,
+      "learning_rate": 0.0006119081473277501,
+      "loss": 0.0663,
+      "step": 9400
+    },
+    {
+      "epoch": 0.07906519966043578,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 0.0005904096396634935,
+      "loss": 0.0642,
+      "step": 9500
+    },
+    {
+      "epoch": 0.07989746492001931,
+      "grad_norm": 0.03271484375,
+      "learning_rate": 0.0005691362550590297,
+      "loss": 0.0644,
+      "step": 9600
+    },
+    {
+      "epoch": 0.08072973017960285,
+      "grad_norm": 0.047607421875,
+      "learning_rate": 0.0005480996860000663,
+      "loss": 0.0648,
+      "step": 9700
+    },
+    {
+      "epoch": 0.08156199543918638,
+      "grad_norm": 0.03955078125,
+      "learning_rate": 0.0005273114948114346,
+      "loss": 0.0655,
+      "step": 9800
+    },
+    {
+      "epoch": 0.08239426069876991,
+      "grad_norm": 0.048095703125,
+      "learning_rate": 0.0005067831073020928,
+      "loss": 0.0651,
+      "step": 9900
+    },
+    {
+      "epoch": 0.08322652595835345,
+      "grad_norm": 0.0299072265625,
+      "learning_rate": 0.00048652580648515787,
+      "loss": 0.066,
+      "step": 10000
+    },
+    {
+      "epoch": 0.08322652595835345,
+      "eval_peoplespeech-clean-transcription_loss": 1.489192247390747,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.4911,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.743,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.843,
+      "step": 10000
+    },
+    {
+      "epoch": 0.08405879121793698,
+      "grad_norm": 0.055908203125,
+      "learning_rate": 0.0004665507263764299,
+      "loss": 0.0653,
+      "step": 10100
+    },
+    {
+      "epoch": 0.08489105647752052,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 0.0004468688458748006,
+      "loss": 0.065,
+      "step": 10200
+    },
+    {
+      "epoch": 0.08572332173710405,
+      "grad_norm": 0.05322265625,
+      "learning_rate": 0.0004274909827279283,
+      "loss": 0.0654,
+      "step": 10300
+    },
+    {
+      "epoch": 0.08655558699668758,
+      "grad_norm": 0.0419921875,
+      "learning_rate": 0.0004084277875864776,
+      "loss": 0.0648,
+      "step": 10400
+    },
+    {
+      "epoch": 0.08738785225627112,
+      "grad_norm": 0.050537109375,
+      "learning_rate": 0.00038968973815020803,
+      "loss": 0.0643,
+      "step": 10500
+    },
+    {
+      "epoch": 0.08822011751585465,
+      "grad_norm": 0.0458984375,
+      "learning_rate": 0.00037128713340911534,
+      "loss": 0.0669,
+      "step": 10600
+    },
+    {
+      "epoch": 0.08905238277543819,
+      "grad_norm": 0.05029296875,
+      "learning_rate": 0.00035323008798280133,
+      "loss": 0.0657,
+      "step": 10700
+    },
+    {
+      "epoch": 0.08988464803502172,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 0.00033552852656117837,
+      "loss": 0.0646,
+      "step": 10800
+    },
+    {
+      "epoch": 0.09071691329460525,
+      "grad_norm": 0.04248046875,
+      "learning_rate": 0.00031819217844956217,
+      "loss": 0.0636,
+      "step": 10900
+    },
+    {
+      "epoch": 0.09154917855418879,
+      "grad_norm": 0.05029296875,
+      "learning_rate": 0.00030123057222115836,
+      "loss": 0.0658,
+      "step": 11000
+    },
+    {
+      "epoch": 0.09154917855418879,
+      "eval_peoplespeech-clean-transcription_loss": 1.4850726127624512,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.5339,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.713,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.839,
+      "step": 11000
+    },
+    {
+      "epoch": 0.09238144381377232,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 0.0002846530304798727,
+      "loss": 0.0641,
+      "step": 11100
+    },
+    {
+      "epoch": 0.09321370907335585,
+      "grad_norm": 0.048095703125,
+      "learning_rate": 0.00026846866473633125,
+      "loss": 0.065,
+      "step": 11200
+    },
+    {
+      "epoch": 0.09404597433293939,
+      "grad_norm": 0.0458984375,
+      "learning_rate": 0.00025268637039992293,
+      "loss": 0.065,
+      "step": 11300
+    },
+    {
+      "epoch": 0.09487823959252292,
+      "grad_norm": 0.045166015625,
+      "learning_rate": 0.00023731482188961818,
+      "loss": 0.0654,
+      "step": 11400
+    },
+    {
+      "epoch": 0.09571050485210646,
+      "grad_norm": 0.04150390625,
+      "learning_rate": 0.00022236246786624792,
+      "loss": 0.0661,
+      "step": 11500
+    },
+    {
+      "epoch": 0.09654277011169,
+      "grad_norm": 0.036376953125,
+      "learning_rate": 0.00020783752658887068,
+      "loss": 0.066,
+      "step": 11600
+    },
+    {
+      "epoch": 0.09737503537127354,
+      "grad_norm": 0.044189453125,
+      "learning_rate": 0.0001937479813977703,
+      "loss": 0.0649,
+      "step": 11700
+    },
+    {
+      "epoch": 0.09820730063085707,
+      "grad_norm": 0.037353515625,
+      "learning_rate": 0.00018010157632657541,
+      "loss": 0.0636,
+      "step": 11800
+    },
+    {
+      "epoch": 0.0990395658904406,
+      "grad_norm": 0.0341796875,
+      "learning_rate": 0.00016690581184590858,
+      "loss": 0.0649,
+      "step": 11900
+    },
+    {
+      "epoch": 0.09987183115002414,
+      "grad_norm": 0.052001953125,
+      "learning_rate": 0.00015416794074090258,
+      "loss": 0.0663,
+      "step": 12000
+    },
+    {
+      "epoch": 0.09987183115002414,
+      "eval_peoplespeech-clean-transcription_loss": 1.485172986984253,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.8864,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.474,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.809,
+      "step": 12000
+    },
+    {
+      "epoch": 0.10070409640960767,
+      "grad_norm": 0.0400390625,
+      "learning_rate": 0.00014189496412485593,
+      "loss": 0.0634,
+      "step": 12100
+    },
+    {
+      "epoch": 0.10153636166919121,
+      "grad_norm": 0.049072265625,
+      "learning_rate": 0.00013009362759120978,
+      "loss": 0.0641,
+      "step": 12200
+    },
+    {
+      "epoch": 0.10236862692877474,
+      "grad_norm": 0.054931640625,
+      "learning_rate": 0.00011877041750597173,
+      "loss": 0.0635,
+      "step": 12300
+    },
+    {
+      "epoch": 0.10320089218835828,
+      "grad_norm": 0.041015625,
+      "learning_rate": 0.00010793155744261352,
+      "loss": 0.0656,
+      "step": 12400
+    },
+    {
+      "epoch": 0.10403315744794181,
+      "grad_norm": 0.042724609375,
+      "learning_rate": 9.758300476141169e-05,
+      "loss": 0.0658,
+      "step": 12500
+    },
+    {
+      "epoch": 0.10486542270752534,
+      "grad_norm": 0.046875,
+      "learning_rate": 8.773044733510338e-05,
+      "loss": 0.0655,
+      "step": 12600
+    },
+    {
+      "epoch": 0.10569768796710888,
+      "grad_norm": 0.052001953125,
+      "learning_rate": 7.837930042266262e-05,
+      "loss": 0.0652,
+      "step": 12700
+    },
+    {
+      "epoch": 0.10652995322669241,
+      "grad_norm": 0.044677734375,
+      "learning_rate": 6.953470369291348e-05,
+      "loss": 0.0652,
+      "step": 12800
+    },
+    {
+      "epoch": 0.10736221848627595,
+      "grad_norm": 0.04443359375,
+      "learning_rate": 6.120151839961363e-05,
+      "loss": 0.0639,
+      "step": 12900
+    },
+    {
+      "epoch": 0.10819448374585948,
+      "grad_norm": 0.03955078125,
+      "learning_rate": 5.338432470956589e-05,
+      "loss": 0.0648,
+      "step": 13000
+    },
+    {
+      "epoch": 0.10819448374585948,
+      "eval_peoplespeech-clean-transcription_loss": 1.4848434925079346,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.5215,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.722,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.84,
+      "step": 13000
+    },
+    {
+      "epoch": 0.10902674900544301,
+      "grad_norm": 0.04541015625,
+      "learning_rate": 4.6087419185220966e-05,
+      "loss": 0.0632,
+      "step": 13100
+    },
+    {
+      "epoch": 0.10985901426502655,
+      "grad_norm": 0.050048828125,
+      "learning_rate": 3.931481242315993e-05,
+      "loss": 0.0624,
+      "step": 13200
+    },
+    {
+      "epoch": 0.11069127952461008,
+      "grad_norm": 0.0439453125,
+      "learning_rate": 3.307022684974936e-05,
+      "loss": 0.0649,
+      "step": 13300
+    },
+    {
+      "epoch": 0.11152354478419362,
+      "grad_norm": 0.047119140625,
+      "learning_rate": 2.7357094675186987e-05,
+      "loss": 0.0644,
+      "step": 13400
+    },
+    {
+      "epoch": 0.11235581004377715,
+      "grad_norm": 0.042236328125,
+      "learning_rate": 2.2178556007054874e-05,
+      "loss": 0.0644,
+      "step": 13500
+    },
+    {
+      "epoch": 0.11318807530336068,
+      "grad_norm": 0.03564453125,
+      "learning_rate": 1.7537457124423894e-05,
+      "loss": 0.0664,
+      "step": 13600
+    },
+    {
+      "epoch": 0.11402034056294422,
+      "grad_norm": 0.05859375,
+      "learning_rate": 1.3436348913453578e-05,
+      "loss": 0.0653,
+      "step": 13700
+    },
+    {
+      "epoch": 0.11485260582252775,
+      "grad_norm": 0.046875,
+      "learning_rate": 9.877485465349056e-06,
+      "loss": 0.0646,
+      "step": 13800
+    },
+    {
+      "epoch": 0.11568487108211128,
+      "grad_norm": 0.030517578125,
+      "learning_rate": 6.862822837445881e-06,
+      "loss": 0.0663,
+      "step": 13900
+    },
+    {
+      "epoch": 0.11651713634169482,
+      "grad_norm": 0.03759765625,
+      "learning_rate": 4.394017978101905e-06,
+      "loss": 0.0648,
+      "step": 14000
+    },
+    {
+      "epoch": 0.11651713634169482,
+      "eval_peoplespeech-clean-transcription_loss": 1.4849343299865723,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.3501,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.845,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.856,
+      "step": 14000
+    },
+    {
+      "epoch": 0.11734940160127835,
+      "grad_norm": 0.045166015625,
+      "learning_rate": 2.472427815989886e-06,
+      "loss": 0.0636,
+      "step": 14100
+    },
+    {
+      "epoch": 0.1181816668608619,
+      "grad_norm": 0.0361328125,
+      "learning_rate": 1.099108514288627e-06,
+      "loss": 0.0646,
+      "step": 14200
+    },
+    {
+      "epoch": 0.11901393212044543,
+      "grad_norm": 0.04052734375,
+      "learning_rate": 2.748148901841052e-07,
+      "loss": 0.0645,
+      "step": 14300
+    },
+    {
+      "epoch": 0.11984619738002897,
+      "grad_norm": 0.056396484375,
+      "learning_rate": 0.0,
+      "loss": 0.0661,
+      "step": 14400
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 14400,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 3600,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.254469835026432e+17,
+  "train_batch_size": 24,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-14400/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37b86db64b66888697120b24a102f1ab62ab1d7da7b6bd4cd2e497d4499c3fb5
+size 5688

checkpoint-14400/ultravox_config.py

@@ -0,0 +1,173 @@
+import dataclasses
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+import transformers
+
+
+@dataclasses.dataclass
+class LoraConfigSimplified:
+    """
+    Low Rank Approximation (LoRA) configuration.
+
+    Used for language and audio models separately.
+    """
+
+    # The rank of the approximation
+    r: int = 0
+    lora_alpha: float = 8
+    target_modules: Optional[List[str]] = dataclasses.field(
+        default_factory=lambda: ["k_proj", "q_proj", "linear_k", "linear_q"]
+    )
+    # A list of module names regex patterns to unfreeze. Only used if r == 0.
+    unfreeze_layers: Optional[List[str]] = None
+
+
+class LossFunction(str, Enum):
+    CrossEntropy = "ce"
+    KL_Divergence = "kl"
+
+
+@dataclasses.dataclass
+class LossConfig:
+    loss_function: LossFunction = LossFunction.CrossEntropy
+    kl_temperature: float = 2.0
+
+    @property
+    def requires_alt_fields(self):
+        return self.loss_function == LossFunction.KL_Divergence
+
+
+class UltravoxConfig(transformers.PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`UltravoxForConditionalGeneration`]. It is used to instantiate an
+    Ultravox model according to the specified arguments, defining the model architecture.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        audio_config (`Wav2Vec2Config`,  *optional*):
+            Custom audio config or dict
+        text_config (`Union[AutoConfig, dict]`, *optional*):
+            The config object of the text backbone. Can be any of `LlamaConfig` or `MistralConfig`.
+        ignore_index (`int`, *optional*, defaults to -100):
+            The ignore index for the loss function.
+        audio_token_index (`int`, *optional*, defaults to 32000):
+            The audio token index to encode the audio prompt.
+        stack_factor (`int`, *optional*, defaults to 8):
+            Audio downsampling factor for the multimodal projector.
+        norm_init (`float`, *optional*, defaults to 0.4):
+            The initialization value for the layer normalization.
+        projector_act (`str`, *optional*, defaults to `"swiglu"`):
+            The activation function used by the multimodal projector.
+        text_model_lora_config (`LoraConfigSimplified`, *optional*):
+            The LoRA configuration for finetuning the text model.
+        audio_model_lora_config (`LoraConfigSimplified`, *optional*):
+            The LoRA configuration for finetuning the audio model.
+        audio_latency_block_size (`int`, *optional*, defaults to `None`):
+            The latency block size for simulating audio streaming.
+
+
+    Example:
+
+    ```python
+    >>> from transformers import UltravoxModel, Wav2Vec2Config, UltravoxConfig, LlamaConfig
+
+    >>> # Initializing an audio encoder config
+    >>> audio_config = Wav2Vec2Config()
+
+    >>> # Initializing a Llama config
+    >>> text_config = LlamaConfig()
+
+    >>> # Initializing a default configuration
+    >>> configuration = UltravoxConfig(audio_config, text_config)
+
+    >>> # Initializing a completely untrained model from the configuration
+    >>> model = UltravoxModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+
+    >>> # Initialize a model from pretrained checkpoints and random projector weights
+    >>> config = UltravoxConfig(audio_model_id="facebook/wav2vec2-base-960h", text_model_id="meta-llama/Llama-2-7b-chat-hf")
+    ```"""
+
+    model_type = "ultravox"
+    is_composition = False
+
+    def __init__(
+        self,
+        audio_config: Optional[Dict[str, Any]] = None,
+        text_config: Optional[Dict[str, Any]] = None,
+        audio_model_id: Optional[str] = None,
+        text_model_id: Optional[str] = None,
+        ignore_index: int = -100,
+        hidden_size: int = 4096,
+        stack_factor: int = 8,
+        norm_init: float = 0.4,
+        projector_act: str = "swiglu",
+        projector_ln_mid: bool = False,  # defaults to False for compatibility with v0.4.1 and below
+        text_model_lora_config: Optional[LoraConfigSimplified] = None,
+        audio_model_lora_config: Optional[LoraConfigSimplified] = None,
+        audio_latency_block_size: Optional[int] = None,
+        **kwargs,
+    ):
+        self.ignore_index = ignore_index
+
+        self.audio_model_id = audio_model_id
+        self.text_model_id = text_model_id
+
+        self.hidden_size = hidden_size
+        self.stack_factor = stack_factor
+        self.norm_init = norm_init
+        self.projector_act = projector_act
+        self.projector_ln_mid = projector_ln_mid
+        if text_model_id is not None:
+            self.text_config: transformers.LlamaConfig = (
+                transformers.AutoConfig.from_pretrained(text_model_id)
+            )
+        else:
+            text_config = text_config or {}
+            self.text_config = transformers.CONFIG_MAPPING[
+                text_config.get("model_type", "llama")
+            ](**text_config)
+
+        if audio_model_id is not None:
+            self.audio_config: transformers.PretrainedConfig = (
+                transformers.AutoConfig.from_pretrained(audio_model_id)
+            )
+        else:
+            audio_config = audio_config or {}
+            self.audio_config = transformers.CONFIG_MAPPING[
+                audio_config.get("model_type", "wav2vec2")
+            ](**audio_config)
+
+        self.text_model_lora_config = (
+            text_model_lora_config
+            if isinstance(text_model_lora_config, dict)
+            else dataclasses.asdict(text_model_lora_config or LoraConfigSimplified())
+        )
+        self.audio_model_lora_config = (
+            audio_model_lora_config
+            if isinstance(audio_model_lora_config, dict)
+            else dataclasses.asdict(audio_model_lora_config or LoraConfigSimplified())
+        )
+        self.audio_latency_block_size = audio_latency_block_size
+
+        self.vocab_size = self.text_config.vocab_size
+
+        self.initializer_range = self.text_config.initializer_range
+
+        super().__init__(**kwargs)
+
+    def to_diff_dict(self) -> Dict[str, Any]:
+        diff_dict = super().to_diff_dict()
+
+        # remove text_config and audio_config if text_model_id and audio_model_id are present
+        if self.text_model_id is not None:
+            diff_dict.pop("text_config", None)
+        if self.audio_model_id is not None:
+            diff_dict.pop("audio_config", None)
+
+        return diff_dict

checkpoint-14400/ultravox_model.py

@@ -0,0 +1,754 @@
+import logging
+import re
+from typing import Any, Dict, Optional, Set, Tuple, Union
+
+import peft
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import transformers
+import transformers.activations
+import transformers.modeling_outputs
+import transformers.models
+from transformers.models.whisper import modeling_whisper as whisper
+
+# We must use relative import in this directory to allow uploading to HF Hub
+# Even "from . import X" pattern doesn't work (undocumented and unclear why)
+from .ultravox_config import LossConfig
+from .ultravox_config import LossFunction
+from .ultravox_config import UltravoxConfig
+
+
+class UltravoxModel(transformers.LlamaPreTrainedModel):
+    """
+    The Ultravox model which consists of an audio encoder and a language model.
+
+    Audio input is processed by the audio encoder, then every `stack_factor` frames are stacked together and
+    projected to the language model's embedding space using a few linear layers.
+    The text is embedded by the language model as usual and then the audio and text embeddings are merged together.
+
+    A special token `<|audio|>` is used to indicate the start of the audio embeddings in the merged embeddings.
+
+    Parameters:
+        config: Model configuration class with all the parameters of the model.
+    """
+
+    config_class = UltravoxConfig
+    config: UltravoxConfig  # for type hinting
+    # Usually we load encoder and LLM weights from a pretrained model separately, so they are allowed to be missing
+    _keys_to_ignore_on_load_missing = ["audio_tower.*", "language_model.*"]
+    # Since we have kwargs in forward, we need to set this to False, otherwise grad_accum_steps will cause incorrect train loss to be reported
+    # see https://github.com/huggingface/transformers/issues/35856 and https://github.com/huggingface/trl/pull/2615/files
+    accepts_loss_kwargs = False
+
+    def __init__(self, config: UltravoxConfig):
+        super().__init__(config)
+        self._register_load_state_dict_pre_hook(self._pre_load_state_dict_hook)
+
+        self.keep_params: Set[str] = set()
+        self.vocab_size = config.vocab_size
+
+        self.audio_tower = self._create_audio_tower(config)
+        self.multi_modal_projector = self._create_multi_modal_projector(config)
+        self.language_model = self._create_language_model(config)
+
+        # Determine no_split_modules dynamically to use with FSDP auto_wrap policy.
+        # FSDP throws an error if some of the layer types are not found in the model.
+        # This would be something like ["LlamaDecoderLayer", "WhisperEncoderLayer"]
+        self._no_split_modules = (self.language_model._no_split_modules or []) + (
+            self.audio_tower._no_split_modules or []
+        )
+
+        self.loss_config = LossConfig()
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.language_model.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.language_model.get_output_embeddings()
+
+    def set_output_embeddings(self, new_embeddings):
+        self.language_model.set_output_embeddings(new_embeddings)
+
+    def set_decoder(self, decoder):
+        self.language_model.set_decoder(decoder)
+
+    def get_decoder(self):
+        return self.language_model.get_decoder()
+
+    def tie_weights(self):
+        return self.language_model.tie_weights()
+
+    def set_loss_config(self, loss_config: LossConfig):
+        self.loss_config = loss_config
+
+    def _setup_cache(
+        self, cache_cls, max_batch_size: int, max_cache_len: Optional[int] = None
+    ):
+        self.language_model._setup_cache(cache_cls, max_batch_size, max_cache_len)
+
+    def _reorder_cache(self, past_key_values, beam_idx):
+        return self.language_model._reorder_cache(past_key_values, beam_idx)
+
+    def resize_token_embeddings(
+        self,
+        new_num_tokens: Optional[int] = None,
+        pad_to_multiple_of: Optional[int] = None,
+    ) -> nn.Embedding:
+        model_embeds = self.language_model.resize_token_embeddings(
+            new_num_tokens, pad_to_multiple_of
+        )
+        # update vocab size
+        self.config.text_config.vocab_size = model_embeds.num_embeddings
+        self.config.vocab_size = model_embeds.num_embeddings
+        self.vocab_size = model_embeds.num_embeddings
+        return model_embeds
+
+    def _compute_kl_loss(
+        self,
+        lm_output: transformers.modeling_outputs.CausalLMOutputWithPast,
+        labels: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]] = None,
+        alt_input_ids: Optional[torch.Tensor] = None,
+        alt_attention_mask: Optional[torch.Tensor] = None,
+        alt_labels: Optional[torch.Tensor] = None,
+        **kwargs,
+    ):
+        # disable gradient computation for the teacher model
+        with torch.no_grad():
+            # compute the teacher (text-only) model's distribution
+            alt_inputs_embeds = self.get_input_embeddings().forward(alt_input_ids)
+            alt_lm_output = self.language_model.forward(
+                inputs_embeds=alt_inputs_embeds,
+                labels=alt_labels,
+                attention_mask=alt_attention_mask,
+                past_key_values=past_key_values,
+                **kwargs,
+            )
+        # compute the KL divergence loss between the two models
+        kl_loss = F.kl_div(
+            F.log_softmax(
+                lm_output.logits[labels != -100] / self.loss_config.kl_temperature,
+                dim=-1,
+            ),
+            F.softmax(
+                alt_lm_output.logits[alt_labels != -100]
+                / self.loss_config.kl_temperature,
+                dim=-1,
+            ),
+            reduction="batchmean",
+        )
+        return {"loss": kl_loss}
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        audio_values: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        audio_token_start_idx: Optional[torch.Tensor] = None,
+        audio_len: Optional[torch.Tensor] = None,
+        audio_token_len: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]] = None,
+        # the alt_* fields are needed for KL divergence loss
+        alt_input_ids: Optional[torch.Tensor] = None,
+        alt_attention_mask: Optional[torch.Tensor] = None,
+        alt_labels: Optional[torch.Tensor] = None,
+        **kwargs,
+    ) -> Union[Tuple, transformers.modeling_outputs.CausalLMOutputWithPast]:
+        """
+        Forward pass for the Ultravox model.
+
+        `input_ids` are the tokenized text input. They are embedded by the language model as usual.
+        `audio_values` are processed by the audio encoder and then every `stack_factor` frames are stacked together and
+        projected to the language model's embedding space using a few linear layers.
+        The audio and text embeddings are merged together. A special token `<|audio|>` is used to indicate the start
+        of the audio embeddings in the merged embeddings.
+
+        Args:
+            input_ids: The tokenized text input.
+            audio_values: The processed audio values.
+            inputs_embeds: The embeddings for the input tokens.
+            labels: The tokenized text labels.
+            attention_mask: The attention mask for the input.
+            position_ids: The position ids for the input.
+            past_key_values: The past key value cache for the language model attention layers.
+            **kwargs: Additional keyword arguments. Passed directly to the language model.
+        """
+        if inputs_embeds is None:
+            # B x T  ->  B x T x D
+            inputs_embeds = self.get_input_embeddings().forward(input_ids)
+
+        if audio_values is not None:
+            assert (
+                audio_token_start_idx is not None and audio_token_len is not None
+            ), "audio_token_start_idx and audio_token_len must be provided if audio_values are provided."
+            assert (
+                len(audio_token_start_idx) == len(audio_token_len) == len(audio_values)
+            ), "audio_token_start_idx, audio_token_len, and audio_values must have the same batch size."
+
+            # B x A/3200 x D
+            audio_tower_output = self.audio_tower.forward(
+                audio_values.to(self.audio_tower.dtype),
+                audio_len=audio_len,
+            ).last_hidden_state
+            audio_tower_output = audio_tower_output.to(inputs_embeds.dtype)
+
+            audio_embeds = self.multi_modal_projector.forward(audio_tower_output)
+
+            # combine audio and text embeddings
+            for i, (audio, start, length) in enumerate(
+                zip(audio_embeds, audio_token_start_idx, audio_token_len)
+            ):
+                length = min(length, audio.shape[0])
+                inputs_embeds[i, start : start + length] = audio[:length]
+
+        lm_output = self.language_model.forward(
+            inputs_embeds=inputs_embeds,
+            labels=labels,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            **kwargs,
+        )
+        if self.training:
+            if self.loss_config.loss_function == LossFunction.CrossEntropy:
+                return lm_output
+            elif self.loss_config.loss_function == LossFunction.KL_Divergence:
+                return self._compute_kl_loss(
+                    lm_output=lm_output,
+                    labels=labels,
+                    past_key_values=past_key_values,
+                    alt_input_ids=alt_input_ids,
+                    alt_attention_mask=alt_attention_mask,
+                    alt_labels=alt_labels,
+                    **kwargs,
+                )
+            else:
+                raise ValueError(
+                    f"Unsupported loss function: {self.loss_config.loss_function}"
+                )
+        else:
+            return lm_output
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids: torch.Tensor,
+        audio_values: Optional[torch.FloatTensor] = None,
+        audio_token_start_idx: Optional[torch.Tensor] = None,
+        audio_token_len: Optional[torch.Tensor] = None,
+        audio_len: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        cache_position: Optional[torch.Tensor] = None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        model_input = self.language_model.prepare_inputs_for_generation(
+            input_ids=input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            inputs_embeds=inputs_embeds,
+            cache_position=cache_position,
+            **kwargs,
+        )
+
+        # include audio information in model_input only when it is needed during prefilling
+        # audio_token_start_idx should always be relative to the current cache position
+        prefill_start_idx = 0 if cache_position is None else cache_position[0]
+        if (
+            audio_values is not None
+            and audio_token_start_idx is not None
+            and prefill_start_idx <= torch.max(audio_token_start_idx)
+        ):
+            model_input["audio_values"] = audio_values
+            model_input["audio_token_start_idx"] = (
+                audio_token_start_idx - prefill_start_idx
+            )
+            model_input["audio_token_len"] = audio_token_len
+            model_input["audio_len"] = audio_len
+
+        return model_input
+
+    @classmethod
+    def _create_multi_modal_projector(
+        cls, config: UltravoxConfig
+    ) -> "UltravoxProjector":
+        projector = UltravoxProjector(config)
+        projector.to(config.torch_dtype)
+        return projector
+
+    @classmethod
+    def _create_audio_tower(
+        cls, config: UltravoxConfig
+    ) -> Union[transformers.Wav2Vec2Model, "ModifiedWhisperEncoder"]:
+        if config.audio_model_id is not None:
+            if "whisper" in config.audio_model_id.lower():
+                audio_tower = ModifiedWhisperEncoder.from_pretrained(
+                    config.audio_model_id, torch_dtype=config.torch_dtype
+                )
+                audio_tower.init_latency_mask(
+                    config.audio_latency_block_size, dtype=config.torch_dtype
+                )
+            else:
+                assert config.audio_latency_block_size in (
+                    None,
+                    0,
+                ), "only whisper audio tower supports audio latency masking, got non-zero value for 'audio_latency_block_size'"
+                audio_tower = transformers.AutoModel.from_pretrained(
+                    config.audio_model_id, torch_dtype=config.torch_dtype
+                )
+        else:
+            if "whisper" in config.audio_config._name_or_path.lower():
+                audio_tower = ModifiedWhisperEncoder(config.audio_config)
+                audio_tower.init_latency_mask(
+                    config.audio_latency_block_size, dtype=config.torch_dtype
+                )
+            else:
+                assert config.audio_latency_block_size in (
+                    None,
+                    0,
+                ), "only whisper audio tower supports audio latency masking, got non-zero value for 'audio_latency_block_size'"
+                with transformers.modeling_utils.no_init_weights():
+                    # we only ever use from_config if the weights are retrained, hence initializing is not
+                    # required. This makes the model quite creation faster since init on CPU is quite slow.
+                    audio_tower = transformers.AutoModel.from_config(
+                        config.audio_config
+                    )
+
+        if isinstance(
+            audio_tower,
+            (transformers.Wav2Vec2BertModel, transformers.WhisperModel),
+        ):
+            # For these models we only need the encoder part
+            # Wav2Vec2BertModel -> Wav2Vec2BertEncoder
+            # WhisperModel -> WhisperEncoder
+            audio_tower = audio_tower.encoder
+
+        audio_tower = apply_lora(audio_tower, config.audio_model_lora_config)
+        return audio_tower
+
+    @classmethod
+    def _create_language_model(
+        cls, config: UltravoxConfig
+    ) -> transformers.LlamaForCausalLM:
+        if config.text_model_id is not None:
+            language_model = transformers.AutoModelForCausalLM.from_pretrained(
+                config.text_model_id,
+                attn_implementation=config._attn_implementation,
+                torch_dtype=config.torch_dtype,
+            )
+        else:
+            with transformers.modeling_utils.no_init_weights():
+                # we only ever use from_config if the weights are retrained, hence initializing is not
+                # required. This makes the model quite creation faster since init on CPU is quite slow.
+                language_model = transformers.AutoModelForCausalLM.from_config(
+                    config.text_config,
+                    attn_implementation=config._attn_implementation,
+                    torch_dtype=config.torch_dtype,
+                )
+
+        language_model = apply_lora(language_model, config.text_model_lora_config)
+        return language_model
+
+    def merge_and_unload(self):
+        if isinstance(self.language_model, peft.PeftModel):
+            self.language_model = self.language_model.merge_and_unload()
+            # no need to download base language model weights anymore, so we can remove the id
+            self.config.text_model_id = None
+            self.keep_params.update(
+                set(
+                    [
+                        f"language_model.{name}"
+                        for name, _ in self.language_model.named_parameters()
+                    ]
+                )
+            )
+
+        if isinstance(self.audio_tower, peft.PeftModel):
+            self.audio_tower = self.audio_tower.merge_and_unload()
+            # no need to download base audio model weights anymore, so we can remove the id
+            self.config.audio_model_id = None
+            self.keep_params.update(
+                set(
+                    [
+                        f"audio_tower.{name}"
+                        for name, _ in self.audio_tower.named_parameters()
+                    ]
+                )
+            )
+
+        for param in ["text_model_lora_config", "audio_model_lora_config"]:
+            if hasattr(self.config, param):
+                delattr(self.config, param)
+
+    def push_to_hub(self, *args, **kwargs):
+        self.merge_and_unload()
+        return super().push_to_hub(*args, **kwargs)
+
+    def diff_state_dict(
+        self, state_dict: Optional[Dict[str, Any]] = None
+    ) -> Dict[str, Any]:
+        if state_dict is None:
+            state_dict = super().state_dict()
+
+        named_params = dict(self.named_parameters())
+
+        state_dict = {
+            k: v
+            for k, v in state_dict.items()
+            if k in self.keep_params
+            or (k in named_params and named_params[k].requires_grad)
+        }
+
+        return state_dict
+
+    def save_pretrained(
+        self, *args, state_dict: Optional[Dict[str, Any]] = None, **kwargs
+    ):
+        state_dict = self.diff_state_dict(state_dict)
+
+        super().save_pretrained(*args, state_dict=state_dict, **kwargs)
+
+    def _pre_load_state_dict_hook(self, state_dict: Dict[str, Any], *args, **kwargs):
+        self.keep_params.update(set(state_dict.keys()))
+
+    def print_trainable_parameters(self):
+        """
+        Prints the number of trainable parameters in the model (reuses Peft model's method)
+        """
+        count_params = peft.peft_model.PeftModel.get_nb_trainable_parameters
+
+        trainable_params, all_param = count_params(self)
+
+        logging.info(
+            f"trainable params: {trainable_params:,d} || all params: {all_param:,d}"
+            f" || trainable%: {100 * trainable_params / all_param:.1f}%"
+        )
+
+        lm_trainable_params, lm_all_params = count_params(self.language_model)
+        audio_trainable_params, audio_all_params = count_params(self.audio_tower)
+
+        projector_trainable_params = (
+            trainable_params - lm_trainable_params - audio_trainable_params
+        )
+        projector_all_params = all_param - lm_all_params - audio_all_params
+
+        logging.info(
+            f"Trainable%:   "
+            f" LLM: {100 * lm_trainable_params / lm_all_params:.1f}%"
+            f" || Audio Encoder: {100 * audio_trainable_params / audio_all_params:.1f}%"
+            f" || Projector: {100 * projector_trainable_params / projector_all_params:.1f}%"
+        )
+
+
+# TODO: refactor common parts to a shared module
+def is_cache_empty(
+    past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]]
+) -> bool:
+    """
+    Check if the cache is empty.
+    """
+    if past_key_values is None:
+        return True
+    if isinstance(past_key_values, tuple):
+        return all(len(c) == 0 for c in past_key_values)
+    return past_key_values.get_seq_length() == 0
+
+
+def apply_lora(model: torch.nn.Module, lora_config: dict) -> torch.nn.Module:
+    """
+    Applies LoRA finetuning to the model. If the `r` parameter is set to 0, the model is frozen instead.
+    """
+    unfreeze_layers = lora_config.pop("unfreeze_layers", None)
+    lora_config = peft.LoraConfig(**lora_config or {})
+
+    if lora_config.r == 0:
+        # freeze the model entirely, except for the specified layers
+        for name, param in model.named_parameters():
+            if not unfreeze_layers or not any(
+                re.match(layer, name) for layer in unfreeze_layers
+            ):
+                param.requires_grad = False
+            else:
+                logging.info(f"Unfreezing layer: {name} with #{param.numel()} params")
+    else:
+        model = peft.get_peft_model(model, lora_config)
+
+    return model
+
+
+class StackAudioFrames(nn.Module):
+    """
+    Stack the audio embedding frames to reduce the sequence length by a factor of `stack_factor`.
+
+    The number of output frames will be `ceil(T / stack_factor) + 1` where `T` is the number of input frames.
+    NOTE: the extra +1 is intentional: in case the number of audio tokens are over-estimated by the processor,
+    we want to make sure `processor.audio_token_replacement` (i.e. EOS) doesn't get leaked into the middle of embeddings.
+    In most cases this extra padding will get removed in the model's forward function so it has no effect.
+    """
+
+    def __init__(self, stack_factor: int = 8):
+        super().__init__()
+        self.stack_factor = stack_factor
+
+    def forward(self, audio_embeds: torch.Tensor) -> torch.Tensor:
+        B, T, C = audio_embeds.shape
+        T_pad = (T + self.stack_factor - 1) // self.stack_factor * self.stack_factor
+        audio_embeds = F.pad(audio_embeds, (0, 0, 0, T_pad - T + self.stack_factor))
+        B, T, C = audio_embeds.shape
+        audio_embeds = audio_embeds.view(
+            B, T // self.stack_factor, C * self.stack_factor
+        )
+        return audio_embeds
+
+
+class RMSNorm(transformers.models.llama.modeling_llama.LlamaRMSNorm):
+    def __init__(self, hidden_size: int, init: float = 1, eps: float = 1e-6):
+        super().__init__(hidden_size=hidden_size, eps=eps)
+        self.weight.data.fill_(init)
+
+
+class SwiGLU(nn.Module):
+    def forward(self, x):
+        x, gate = x.chunk(2, dim=-1)
+        return F.silu(gate) * x
+
+
+class UltravoxProjector(nn.Module):
+    def __init__(self, config: UltravoxConfig):
+        super().__init__()
+        self.hidden_dim = config.hidden_size
+        self._pad_and_stack = StackAudioFrames(config.stack_factor)
+        dim_in = config.audio_config.hidden_size * config.stack_factor
+        self.ln_pre = RMSNorm(dim_in, init=config.norm_init)
+        self.linear_1 = nn.Linear(dim_in, self.hidden_dim, bias=False)
+        dim_mid = self.hidden_dim
+        self.act = transformers.activations.get_activation(config.projector_act)
+        dim_mid = dim_mid // 2 if config.projector_act == "swiglu" else dim_mid
+        dim_out = config.text_config.hidden_size
+        self.linear_2 = nn.Linear(dim_mid, dim_out, bias=False)
+
+        # Ultravox v0.4.1 and below uses layer_norm after the second linear layer,
+        # while v0.5.0 and above uses layer_norm after the first linear layer.
+        if config.projector_ln_mid:
+            self.ln_mid: nn.Module = RMSNorm(dim_mid, init=config.norm_init)
+            self.ln_post: nn.Module = nn.Identity()
+        else:
+            self.ln_mid = nn.Identity()
+            self.ln_post = RMSNorm(dim_out, init=config.norm_init)
+
+    def forward(self, audio_features: torch.Tensor) -> torch.Tensor:
+        audio_features = self._pad_and_stack(audio_features)
+        audio_features = self.ln_pre(audio_features)
+        hidden_states = self.linear_1(audio_features)
+        hidden_states = self.act(hidden_states)
+        hidden_states = self.ln_mid(hidden_states)
+        hidden_states = self.linear_2(hidden_states)
+        hidden_states = self.ln_post(hidden_states)
+        return hidden_states
+
+
+class ModifiedWhisperEncoder(
+    whisper.WhisperEncoder, transformers.modeling_utils.ModuleUtilsMixin
+):
+    """
+    Encoder portion of OpenAI's Whisper model.
+
+    This implementation is a slightly modified version of HF Transformers' Whisper Encoder, with only a few fixes:
+    1. base_model_prefix updated to allow for doing `.from_pretrained` directly on the encoder
+    2. allow less than 30 second of audio padding to be passed in:
+        - relaxed ValueError check for `input_features` length to be less than or equal to `expected_seq_length` instead of strictly equal
+        - embed_pos is now sliced to match the length of `inputs_embeds`
+
+    Original: https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/modeling_whisper.py
+    """
+
+    base_model_prefix = "model.encoder"
+    _no_split_modules = ["WhisperEncoderLayer"]
+
+    def __init__(self, config: transformers.WhisperConfig):
+        super().__init__(config)
+        self.config.is_decoder = False
+
+    def init_latency_mask(self, audio_latency_block_size: int, dtype: torch.dtype):
+        if audio_latency_block_size is None:
+            self.audio_streaming_mask = None
+            return
+
+        # maximum sequence length
+        max_seqlen = (
+            self.config.max_source_positions
+            * self.conv1.stride[0]
+            * self.conv2.stride[0]
+        )
+        assert (
+            max_seqlen > 0
+        ), f"maximum sequence length must be positive, got {max_seqlen}"
+        assert (
+            max_seqlen % audio_latency_block_size == 0
+        ), f"audio_latency_block_size {audio_latency_block_size} must divide {max_seqlen} evenly."
+        # Given the block size, we calculate number of blocks.
+        audio_latency_nblocks = max_seqlen // audio_latency_block_size
+        audio_streaming_mask = (
+            torch.tril(
+                torch.ones(audio_latency_nblocks, audio_latency_nblocks),
+                diagonal=0,
+            )
+            .repeat_interleave(audio_latency_block_size, dim=0)
+            .repeat_interleave(audio_latency_block_size, dim=1)
+        )
+        audio_streaming_mask = (1.0 - audio_streaming_mask) * torch.finfo(dtype).min
+        audio_streaming_mask = audio_streaming_mask[None, None, :, :]
+        self.register_buffer(
+            "audio_streaming_mask", audio_streaming_mask, persistent=False
+        )
+
+    def forward(
+        self,
+        input_features,
+        audio_len=None,
+        head_mask=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+    ):
+        expected_seq_length = (
+            self.config.max_source_positions
+            * self.conv1.stride[0]
+            * self.conv2.stride[0]
+        )
+        if input_features.shape[-1] > expected_seq_length:
+            raise ValueError(
+                f"Whisper expects the mel input features to be of length {expected_seq_length} or less, but found {input_features.shape[-1]}. Make sure to pad the input mel features to {expected_seq_length}."
+            )
+
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+        inputs_embeds = nn.functional.gelu(self.conv1(input_features))
+        inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds))
+
+        inputs_embeds = inputs_embeds.permute(0, 2, 1)
+        embed_pos = self.embed_positions.weight[: inputs_embeds.size(-2)]
+
+        hidden_states = inputs_embeds + embed_pos
+        hidden_states = nn.functional.dropout(
+            hidden_states, p=self.dropout, training=self.training
+        )
+
+        encoder_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+
+        # Create attention mask based on audio lengths to mask out padding tokens
+        # For each sample in batch:
+        # - Convert raw audio length to feature length after convolutions
+        # - Create boolean mask that is True for valid positions and False for padding
+        # - Convert to extended attention mask format expected by transformer layers
+        #   (1.0 for positions to attend to, large negative for positions to ignore)
+        # This masking ensures consistent behavior between training and inference
+        # by preventing the model from attending to padding tokens in both cases
+        attention_mask = None
+        if audio_len != None:
+            audio_feature_len = self._get_feat_extract_output_lengths(audio_len)
+            max_seq_len = hidden_states.shape[1]
+            attention_mask = torch.arange(max_seq_len, device=hidden_states.device)[
+                None, :
+            ].lt(audio_feature_len.view(-1, 1))
+            attention_mask = self.get_extended_attention_mask(
+                attention_mask,
+                None,
+                device=hidden_states.device,
+                dtype=hidden_states.dtype,
+            )
+
+        if self.audio_streaming_mask is not None:
+            seqlen = hidden_states.size(-2)
+            if attention_mask is not None:
+                attention_mask = torch.minimum(
+                    self.audio_streaming_mask[:, :, :seqlen, :seqlen], attention_mask
+                )  # merge
+            else:
+                attention_mask = self.audio_streaming_mask[:, :, :seqlen, :seqlen]
+            attention_mask = attention_mask.to(hidden_states.dtype)
+
+        # check if head_mask has a correct number of layers specified if desired
+        if head_mask is not None:
+            assert head_mask.size()[0] == (
+                len(self.layers)
+            ), f"The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.size()[0]}."
+
+        for idx, encoder_layer in enumerate(self.layers):
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
+            to_drop = False
+            if self.training:
+                dropout_probability = torch.rand([])
+                if dropout_probability < self.layerdrop:  # skip the layer
+                    to_drop = True
+
+            if to_drop:
+                layer_outputs = (None, None)
+            else:
+                if self.gradient_checkpointing and self.training:
+                    layer_outputs = self._gradient_checkpointing_func(
+                        encoder_layer.__call__,
+                        hidden_states,
+                        attention_mask,
+                        (head_mask[idx] if head_mask is not None else None),
+                        output_attentions,
+                    )
+                else:
+                    layer_outputs = encoder_layer(
+                        hidden_states,
+                        attention_mask,
+                        layer_head_mask=(
+                            head_mask[idx] if head_mask is not None else None
+                        ),
+                        output_attentions=output_attentions,
+                    )
+
+                hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+        hidden_states = self.layer_norm(hidden_states)
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [hidden_states, encoder_states, all_attentions]
+                if v is not None
+            )
+        return transformers.modeling_outputs.BaseModelOutput(
+            last_hidden_state=hidden_states,
+            hidden_states=encoder_states,
+            attentions=all_attentions,
+        )
+
+
+UltravoxConfig.register_for_auto_class()
+UltravoxModel.register_for_auto_class()
+
+transformers.AutoConfig.register("ultravox", UltravoxConfig)
+transformers.AutoModel.register(UltravoxConfig, UltravoxModel)
+
+transformers.activations.ACT2FN["swiglu"] = SwiGLU

checkpoint-3600/config.json

@@ -0,0 +1,44 @@
+{
+  "architectures": [
+    "UltravoxModel"
+  ],
+  "audio_latency_block_size": null,
+  "audio_model_id": "openai/whisper-large-v3-turbo",
+  "audio_model_lora_config": {
+    "lora_alpha": 8,
+    "r": 0,
+    "target_modules": [
+      "k_proj",
+      "q_proj",
+      "linear_k",
+      "linear_q"
+    ]
+  },
+  "auto_map": {
+    "AutoConfig": "ultravox_config.UltravoxConfig",
+    "AutoModel": "ultravox_model.UltravoxModel"
+  },
+  "hidden_size": 4096,
+  "ignore_index": -100,
+  "initializer_range": 0.02,
+  "model_type": "ultravox",
+  "norm_init": 0.4,
+  "pad_token_id": 2,
+  "projector_act": "swiglu",
+  "projector_ln_mid": false,
+  "stack_factor": 8,
+  "text_model_id": "HuggingFaceTB/SmolLM2-1.7B-Instruct",
+  "text_model_lora_config": {
+    "lora_alpha": 8,
+    "r": 0,
+    "target_modules": [
+      "k_proj",
+      "q_proj",
+      "linear_k",
+      "linear_q"
+    ]
+  },
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.47.0",
+  "vocab_size": 49152
+}

checkpoint-3600/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 2,
+  "transformers_version": "4.47.0"
+}

checkpoint-3600/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1080c1b944294199dcd5962182193892ea86e01ffb3ce2aa01c78826bf009d98
+size 92299736

checkpoint-3600/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7444791357081ea9975d646a7093e47e4505a05bbeaca1d147d88d865809af4c
+size 184602962

checkpoint-3600/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb9a3b64cb4cc2afb67461c1d3860326ef6cddab5e1dd39123b1af5d51fc5f9e
+size 14244

checkpoint-3600/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04d44c72cd3a348b42894fd1f0aa67e10dd8fbbcc70558ee71674069905052af
+size 1064

checkpoint-3600/special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|im_end|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-3600/tokenizer_config.json

@@ -0,0 +1,155 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|im_end|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

checkpoint-3600/trainer_state.json

@@ -0,0 +1,319 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.029961549345007242,
+  "eval_steps": 1000,
+  "global_step": 3600,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 8.322652595835345e-06,
+      "grad_norm": 14.9375,
+      "learning_rate": 2e-06,
+      "loss": 0.6484,
+      "step": 1
+    },
+    {
+      "epoch": 0.0008322652595835345,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.0002,
+      "loss": 0.4345,
+      "step": 100
+    },
+    {
+      "epoch": 0.001664530519167069,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 0.0004,
+      "loss": 0.2537,
+      "step": 200
+    },
+    {
+      "epoch": 0.0024967957787506035,
+      "grad_norm": 0.55859375,
+      "learning_rate": 0.0006,
+      "loss": 0.2344,
+      "step": 300
+    },
+    {
+      "epoch": 0.003329061038334138,
+      "grad_norm": 0.154296875,
+      "learning_rate": 0.0008,
+      "loss": 0.222,
+      "step": 400
+    },
+    {
+      "epoch": 0.004161326297917672,
+      "grad_norm": 0.126953125,
+      "learning_rate": 0.001,
+      "loss": 0.2142,
+      "step": 500
+    },
+    {
+      "epoch": 0.004993591557501207,
+      "grad_norm": 0.12353515625,
+      "learning_rate": 0.0012,
+      "loss": 0.2011,
+      "step": 600
+    },
+    {
+      "epoch": 0.005825856817084741,
+      "grad_norm": 0.10693359375,
+      "learning_rate": 0.0014,
+      "loss": 0.1897,
+      "step": 700
+    },
+    {
+      "epoch": 0.006658122076668276,
+      "grad_norm": 0.10595703125,
+      "learning_rate": 0.0016,
+      "loss": 0.1775,
+      "step": 800
+    },
+    {
+      "epoch": 0.0074903873362518105,
+      "grad_norm": 0.0859375,
+      "learning_rate": 0.0018000000000000002,
+      "loss": 0.1615,
+      "step": 900
+    },
+    {
+      "epoch": 0.008322652595835344,
+      "grad_norm": 0.083984375,
+      "learning_rate": 0.002,
+      "loss": 0.1444,
+      "step": 1000
+    },
+    {
+      "epoch": 0.008322652595835344,
+      "eval_peoplespeech-clean-transcription_loss": 2.2110702991485596,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.5405,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.708,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.839,
+      "step": 1000
+    },
+    {
+      "epoch": 0.009154917855418878,
+      "grad_norm": 0.0771484375,
+      "learning_rate": 0.001999725185109816,
+      "loss": 0.1323,
+      "step": 1100
+    },
+    {
+      "epoch": 0.009987183115002414,
+      "grad_norm": 0.083984375,
+      "learning_rate": 0.0019989008914857113,
+      "loss": 0.1242,
+      "step": 1200
+    },
+    {
+      "epoch": 0.010819448374585948,
+      "grad_norm": 0.07861328125,
+      "learning_rate": 0.00199752757218401,
+      "loss": 0.1162,
+      "step": 1300
+    },
+    {
+      "epoch": 0.011651713634169482,
+      "grad_norm": 0.07275390625,
+      "learning_rate": 0.001995605982021898,
+      "loss": 0.1128,
+      "step": 1400
+    },
+    {
+      "epoch": 0.012483978893753018,
+      "grad_norm": 0.07666015625,
+      "learning_rate": 0.0019931371771625545,
+      "loss": 0.1094,
+      "step": 1500
+    },
+    {
+      "epoch": 0.013316244153336551,
+      "grad_norm": 0.06201171875,
+      "learning_rate": 0.001990122514534651,
+      "loss": 0.1052,
+      "step": 1600
+    },
+    {
+      "epoch": 0.014148509412920085,
+      "grad_norm": 0.058837890625,
+      "learning_rate": 0.0019865636510865464,
+      "loss": 0.1022,
+      "step": 1700
+    },
+    {
+      "epoch": 0.014980774672503621,
+      "grad_norm": 0.0625,
+      "learning_rate": 0.001982462542875576,
+      "loss": 0.1011,
+      "step": 1800
+    },
+    {
+      "epoch": 0.015813039932087155,
+      "grad_norm": 0.06787109375,
+      "learning_rate": 0.001977821443992945,
+      "loss": 0.0983,
+      "step": 1900
+    },
+    {
+      "epoch": 0.01664530519167069,
+      "grad_norm": 0.0537109375,
+      "learning_rate": 0.001972642905324813,
+      "loss": 0.0975,
+      "step": 2000
+    },
+    {
+      "epoch": 0.01664530519167069,
+      "eval_peoplespeech-clean-transcription_loss": 1.6914767026901245,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.5301,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.716,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.839,
+      "step": 2000
+    },
+    {
+      "epoch": 0.017477570451254223,
+      "grad_norm": 0.05859375,
+      "learning_rate": 0.0019669297731502505,
+      "loss": 0.0947,
+      "step": 2100
+    },
+    {
+      "epoch": 0.018309835710837757,
+      "grad_norm": 0.062255859375,
+      "learning_rate": 0.00196068518757684,
+      "loss": 0.0935,
+      "step": 2200
+    },
+    {
+      "epoch": 0.019142100970421294,
+      "grad_norm": 0.059326171875,
+      "learning_rate": 0.001953912580814779,
+      "loss": 0.0911,
+      "step": 2300
+    },
+    {
+      "epoch": 0.019974366230004828,
+      "grad_norm": 0.060302734375,
+      "learning_rate": 0.0019466156752904343,
+      "loss": 0.0904,
+      "step": 2400
+    },
+    {
+      "epoch": 0.020806631489588362,
+      "grad_norm": 0.0615234375,
+      "learning_rate": 0.0019387984816003866,
+      "loss": 0.0882,
+      "step": 2500
+    },
+    {
+      "epoch": 0.021638896749171896,
+      "grad_norm": 0.056884765625,
+      "learning_rate": 0.0019304652963070869,
+      "loss": 0.0874,
+      "step": 2600
+    },
+    {
+      "epoch": 0.02247116200875543,
+      "grad_norm": 0.049072265625,
+      "learning_rate": 0.0019216206995773372,
+      "loss": 0.0871,
+      "step": 2700
+    },
+    {
+      "epoch": 0.023303427268338964,
+      "grad_norm": 0.053955078125,
+      "learning_rate": 0.0019122695526648968,
+      "loss": 0.0856,
+      "step": 2800
+    },
+    {
+      "epoch": 0.0241356925279225,
+      "grad_norm": 0.05517578125,
+      "learning_rate": 0.0019024169952385887,
+      "loss": 0.0845,
+      "step": 2900
+    },
+    {
+      "epoch": 0.024967957787506035,
+      "grad_norm": 0.0546875,
+      "learning_rate": 0.0018920684425573864,
+      "loss": 0.0852,
+      "step": 3000
+    },
+    {
+      "epoch": 0.024967957787506035,
+      "eval_peoplespeech-clean-transcription_loss": 1.6414048671722412,
+      "eval_peoplespeech-clean-transcription_model_preparation_time": 0.0065,
+      "eval_peoplespeech-clean-transcription_runtime": 9.8151,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 6.521,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.815,
+      "step": 3000
+    },
+    {
+      "epoch": 0.02580022304708957,
+      "grad_norm": 0.052978515625,
+      "learning_rate": 0.0018812295824940284,
+      "loss": 0.0836,
+      "step": 3100
+    },
+    {
+      "epoch": 0.026632488306673103,
+      "grad_norm": 0.05126953125,
+      "learning_rate": 0.0018699063724087904,
+      "loss": 0.0821,
+      "step": 3200
+    },
+    {
+      "epoch": 0.027464753566256637,
+      "grad_norm": 0.056884765625,
+      "learning_rate": 0.0018581050358751443,
+      "loss": 0.0816,
+      "step": 3300
+    },
+    {
+      "epoch": 0.02829701882584017,
+      "grad_norm": 0.045654296875,
+      "learning_rate": 0.0018458320592590974,
+      "loss": 0.0814,
+      "step": 3400
+    },
+    {
+      "epoch": 0.029129284085423705,
+      "grad_norm": 0.04638671875,
+      "learning_rate": 0.0018330941881540914,
+      "loss": 0.0791,
+      "step": 3500
+    },
+    {
+      "epoch": 0.029961549345007242,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 0.0018198984236734246,
+      "loss": 0.0804,
+      "step": 3600
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 14400,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 3600,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.3112799838724096e+17,
+  "train_batch_size": 24,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-3600/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37b86db64b66888697120b24a102f1ab62ab1d7da7b6bd4cd2e497d4499c3fb5
+size 5688

checkpoint-3600/ultravox_config.py

@@ -0,0 +1,173 @@
+import dataclasses
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+import transformers
+
+
+@dataclasses.dataclass
+class LoraConfigSimplified:
+    """
+    Low Rank Approximation (LoRA) configuration.
+
+    Used for language and audio models separately.
+    """
+
+    # The rank of the approximation
+    r: int = 0
+    lora_alpha: float = 8
+    target_modules: Optional[List[str]] = dataclasses.field(
+        default_factory=lambda: ["k_proj", "q_proj", "linear_k", "linear_q"]
+    )
+    # A list of module names regex patterns to unfreeze. Only used if r == 0.
+    unfreeze_layers: Optional[List[str]] = None
+
+
+class LossFunction(str, Enum):
+    CrossEntropy = "ce"
+    KL_Divergence = "kl"
+
+
+@dataclasses.dataclass
+class LossConfig:
+    loss_function: LossFunction = LossFunction.CrossEntropy
+    kl_temperature: float = 2.0
+
+    @property
+    def requires_alt_fields(self):
+        return self.loss_function == LossFunction.KL_Divergence
+
+
+class UltravoxConfig(transformers.PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`UltravoxForConditionalGeneration`]. It is used to instantiate an
+    Ultravox model according to the specified arguments, defining the model architecture.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        audio_config (`Wav2Vec2Config`,  *optional*):
+            Custom audio config or dict
+        text_config (`Union[AutoConfig, dict]`, *optional*):
+            The config object of the text backbone. Can be any of `LlamaConfig` or `MistralConfig`.
+        ignore_index (`int`, *optional*, defaults to -100):
+            The ignore index for the loss function.
+        audio_token_index (`int`, *optional*, defaults to 32000):
+            The audio token index to encode the audio prompt.
+        stack_factor (`int`, *optional*, defaults to 8):
+            Audio downsampling factor for the multimodal projector.
+        norm_init (`float`, *optional*, defaults to 0.4):
+            The initialization value for the layer normalization.
+        projector_act (`str`, *optional*, defaults to `"swiglu"`):
+            The activation function used by the multimodal projector.
+        text_model_lora_config (`LoraConfigSimplified`, *optional*):
+            The LoRA configuration for finetuning the text model.
+        audio_model_lora_config (`LoraConfigSimplified`, *optional*):
+            The LoRA configuration for finetuning the audio model.
+        audio_latency_block_size (`int`, *optional*, defaults to `None`):
+            The latency block size for simulating audio streaming.
+
+
+    Example:
+
+    ```python
+    >>> from transformers import UltravoxModel, Wav2Vec2Config, UltravoxConfig, LlamaConfig
+
+    >>> # Initializing an audio encoder config
+    >>> audio_config = Wav2Vec2Config()
+
+    >>> # Initializing a Llama config
+    >>> text_config = LlamaConfig()
+
+    >>> # Initializing a default configuration
+    >>> configuration = UltravoxConfig(audio_config, text_config)
+
+    >>> # Initializing a completely untrained model from the configuration
+    >>> model = UltravoxModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+
+    >>> # Initialize a model from pretrained checkpoints and random projector weights
+    >>> config = UltravoxConfig(audio_model_id="facebook/wav2vec2-base-960h", text_model_id="meta-llama/Llama-2-7b-chat-hf")
+    ```"""
+
+    model_type = "ultravox"
+    is_composition = False
+
+    def __init__(
+        self,
+        audio_config: Optional[Dict[str, Any]] = None,
+        text_config: Optional[Dict[str, Any]] = None,
+        audio_model_id: Optional[str] = None,
+        text_model_id: Optional[str] = None,
+        ignore_index: int = -100,
+        hidden_size: int = 4096,
+        stack_factor: int = 8,
+        norm_init: float = 0.4,
+        projector_act: str = "swiglu",
+        projector_ln_mid: bool = False,  # defaults to False for compatibility with v0.4.1 and below
+        text_model_lora_config: Optional[LoraConfigSimplified] = None,
+        audio_model_lora_config: Optional[LoraConfigSimplified] = None,
+        audio_latency_block_size: Optional[int] = None,
+        **kwargs,
+    ):
+        self.ignore_index = ignore_index
+
+        self.audio_model_id = audio_model_id
+        self.text_model_id = text_model_id
+
+        self.hidden_size = hidden_size
+        self.stack_factor = stack_factor
+        self.norm_init = norm_init
+        self.projector_act = projector_act
+        self.projector_ln_mid = projector_ln_mid
+        if text_model_id is not None:
+            self.text_config: transformers.LlamaConfig = (
+                transformers.AutoConfig.from_pretrained(text_model_id)
+            )
+        else:
+            text_config = text_config or {}
+            self.text_config = transformers.CONFIG_MAPPING[
+                text_config.get("model_type", "llama")
+            ](**text_config)
+
+        if audio_model_id is not None:
+            self.audio_config: transformers.PretrainedConfig = (
+                transformers.AutoConfig.from_pretrained(audio_model_id)
+            )
+        else:
+            audio_config = audio_config or {}
+            self.audio_config = transformers.CONFIG_MAPPING[
+                audio_config.get("model_type", "wav2vec2")
+            ](**audio_config)
+
+        self.text_model_lora_config = (
+            text_model_lora_config
+            if isinstance(text_model_lora_config, dict)
+            else dataclasses.asdict(text_model_lora_config or LoraConfigSimplified())
+        )
+        self.audio_model_lora_config = (
+            audio_model_lora_config
+            if isinstance(audio_model_lora_config, dict)
+            else dataclasses.asdict(audio_model_lora_config or LoraConfigSimplified())
+        )
+        self.audio_latency_block_size = audio_latency_block_size
+
+        self.vocab_size = self.text_config.vocab_size
+
+        self.initializer_range = self.text_config.initializer_range
+
+        super().__init__(**kwargs)
+
+    def to_diff_dict(self) -> Dict[str, Any]:
+        diff_dict = super().to_diff_dict()
+
+        # remove text_config and audio_config if text_model_id and audio_model_id are present
+        if self.text_model_id is not None:
+            diff_dict.pop("text_config", None)
+        if self.audio_model_id is not None:
+            diff_dict.pop("audio_config", None)
+
+        return diff_dict

checkpoint-3600/ultravox_model.py

@@ -0,0 +1,754 @@
+import logging
+import re
+from typing import Any, Dict, Optional, Set, Tuple, Union
+
+import peft
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import transformers
+import transformers.activations
+import transformers.modeling_outputs
+import transformers.models
+from transformers.models.whisper import modeling_whisper as whisper
+
+# We must use relative import in this directory to allow uploading to HF Hub
+# Even "from . import X" pattern doesn't work (undocumented and unclear why)
+from .ultravox_config import LossConfig
+from .ultravox_config import LossFunction
+from .ultravox_config import UltravoxConfig
+
+
+class UltravoxModel(transformers.LlamaPreTrainedModel):
+    """
+    The Ultravox model which consists of an audio encoder and a language model.
+
+    Audio input is processed by the audio encoder, then every `stack_factor` frames are stacked together and
+    projected to the language model's embedding space using a few linear layers.
+    The text is embedded by the language model as usual and then the audio and text embeddings are merged together.
+
+    A special token `<|audio|>` is used to indicate the start of the audio embeddings in the merged embeddings.
+
+    Parameters:
+        config: Model configuration class with all the parameters of the model.
+    """
+
+    config_class = UltravoxConfig
+    config: UltravoxConfig  # for type hinting
+    # Usually we load encoder and LLM weights from a pretrained model separately, so they are allowed to be missing
+    _keys_to_ignore_on_load_missing = ["audio_tower.*", "language_model.*"]
+    # Since we have kwargs in forward, we need to set this to False, otherwise grad_accum_steps will cause incorrect train loss to be reported
+    # see https://github.com/huggingface/transformers/issues/35856 and https://github.com/huggingface/trl/pull/2615/files
+    accepts_loss_kwargs = False
+
+    def __init__(self, config: UltravoxConfig):
+        super().__init__(config)
+        self._register_load_state_dict_pre_hook(self._pre_load_state_dict_hook)
+
+        self.keep_params: Set[str] = set()
+        self.vocab_size = config.vocab_size
+
+        self.audio_tower = self._create_audio_tower(config)
+        self.multi_modal_projector = self._create_multi_modal_projector(config)
+        self.language_model = self._create_language_model(config)
+
+        # Determine no_split_modules dynamically to use with FSDP auto_wrap policy.
+        # FSDP throws an error if some of the layer types are not found in the model.
+        # This would be something like ["LlamaDecoderLayer", "WhisperEncoderLayer"]
+        self._no_split_modules = (self.language_model._no_split_modules or []) + (
+            self.audio_tower._no_split_modules or []
+        )
+
+        self.loss_config = LossConfig()
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.language_model.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.language_model.get_output_embeddings()
+
+    def set_output_embeddings(self, new_embeddings):
+        self.language_model.set_output_embeddings(new_embeddings)
+
+    def set_decoder(self, decoder):
+        self.language_model.set_decoder(decoder)
+
+    def get_decoder(self):
+        return self.language_model.get_decoder()
+
+    def tie_weights(self):
+        return self.language_model.tie_weights()
+
+    def set_loss_config(self, loss_config: LossConfig):
+        self.loss_config = loss_config
+
+    def _setup_cache(
+        self, cache_cls, max_batch_size: int, max_cache_len: Optional[int] = None
+    ):
+        self.language_model._setup_cache(cache_cls, max_batch_size, max_cache_len)
+
+    def _reorder_cache(self, past_key_values, beam_idx):
+        return self.language_model._reorder_cache(past_key_values, beam_idx)
+
+    def resize_token_embeddings(
+        self,
+        new_num_tokens: Optional[int] = None,
+        pad_to_multiple_of: Optional[int] = None,
+    ) -> nn.Embedding:
+        model_embeds = self.language_model.resize_token_embeddings(
+            new_num_tokens, pad_to_multiple_of
+        )
+        # update vocab size
+        self.config.text_config.vocab_size = model_embeds.num_embeddings
+        self.config.vocab_size = model_embeds.num_embeddings
+        self.vocab_size = model_embeds.num_embeddings
+        return model_embeds
+
+    def _compute_kl_loss(
+        self,
+        lm_output: transformers.modeling_outputs.CausalLMOutputWithPast,
+        labels: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]] = None,
+        alt_input_ids: Optional[torch.Tensor] = None,
+        alt_attention_mask: Optional[torch.Tensor] = None,
+        alt_labels: Optional[torch.Tensor] = None,
+        **kwargs,
+    ):
+        # disable gradient computation for the teacher model
+        with torch.no_grad():
+            # compute the teacher (text-only) model's distribution
+            alt_inputs_embeds = self.get_input_embeddings().forward(alt_input_ids)
+            alt_lm_output = self.language_model.forward(
+                inputs_embeds=alt_inputs_embeds,
+                labels=alt_labels,
+                attention_mask=alt_attention_mask,
+                past_key_values=past_key_values,
+                **kwargs,
+            )
+        # compute the KL divergence loss between the two models
+        kl_loss = F.kl_div(
+            F.log_softmax(
+                lm_output.logits[labels != -100] / self.loss_config.kl_temperature,
+                dim=-1,
+            ),
+            F.softmax(
+                alt_lm_output.logits[alt_labels != -100]
+                / self.loss_config.kl_temperature,
+                dim=-1,
+            ),
+            reduction="batchmean",
+        )
+        return {"loss": kl_loss}
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        audio_values: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        audio_token_start_idx: Optional[torch.Tensor] = None,
+        audio_len: Optional[torch.Tensor] = None,
+        audio_token_len: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]] = None,
+        # the alt_* fields are needed for KL divergence loss
+        alt_input_ids: Optional[torch.Tensor] = None,
+        alt_attention_mask: Optional[torch.Tensor] = None,
+        alt_labels: Optional[torch.Tensor] = None,
+        **kwargs,
+    ) -> Union[Tuple, transformers.modeling_outputs.CausalLMOutputWithPast]:
+        """
+        Forward pass for the Ultravox model.
+
+        `input_ids` are the tokenized text input. They are embedded by the language model as usual.
+        `audio_values` are processed by the audio encoder and then every `stack_factor` frames are stacked together and
+        projected to the language model's embedding space using a few linear layers.
+        The audio and text embeddings are merged together. A special token `<|audio|>` is used to indicate the start
+        of the audio embeddings in the merged embeddings.
+
+        Args:
+            input_ids: The tokenized text input.
+            audio_values: The processed audio values.
+            inputs_embeds: The embeddings for the input tokens.
+            labels: The tokenized text labels.
+            attention_mask: The attention mask for the input.
+            position_ids: The position ids for the input.
+            past_key_values: The past key value cache for the language model attention layers.
+            **kwargs: Additional keyword arguments. Passed directly to the language model.
+        """
+        if inputs_embeds is None:
+            # B x T  ->  B x T x D
+            inputs_embeds = self.get_input_embeddings().forward(input_ids)
+
+        if audio_values is not None:
+            assert (
+                audio_token_start_idx is not None and audio_token_len is not None
+            ), "audio_token_start_idx and audio_token_len must be provided if audio_values are provided."
+            assert (
+                len(audio_token_start_idx) == len(audio_token_len) == len(audio_values)
+            ), "audio_token_start_idx, audio_token_len, and audio_values must have the same batch size."
+
+            # B x A/3200 x D
+            audio_tower_output = self.audio_tower.forward(
+                audio_values.to(self.audio_tower.dtype),
+                audio_len=audio_len,
+            ).last_hidden_state
+            audio_tower_output = audio_tower_output.to(inputs_embeds.dtype)
+
+            audio_embeds = self.multi_modal_projector.forward(audio_tower_output)
+
+            # combine audio and text embeddings
+            for i, (audio, start, length) in enumerate(
+                zip(audio_embeds, audio_token_start_idx, audio_token_len)
+            ):
+                length = min(length, audio.shape[0])
+                inputs_embeds[i, start : start + length] = audio[:length]
+
+        lm_output = self.language_model.forward(
+            inputs_embeds=inputs_embeds,
+            labels=labels,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            **kwargs,
+        )
+        if self.training:
+            if self.loss_config.loss_function == LossFunction.CrossEntropy:
+                return lm_output
+            elif self.loss_config.loss_function == LossFunction.KL_Divergence:
+                return self._compute_kl_loss(
+                    lm_output=lm_output,
+                    labels=labels,
+                    past_key_values=past_key_values,
+                    alt_input_ids=alt_input_ids,
+                    alt_attention_mask=alt_attention_mask,
+                    alt_labels=alt_labels,
+                    **kwargs,
+                )
+            else:
+                raise ValueError(
+                    f"Unsupported loss function: {self.loss_config.loss_function}"
+                )
+        else:
+            return lm_output
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids: torch.Tensor,
+        audio_values: Optional[torch.FloatTensor] = None,
+        audio_token_start_idx: Optional[torch.Tensor] = None,
+        audio_token_len: Optional[torch.Tensor] = None,
+        audio_len: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        cache_position: Optional[torch.Tensor] = None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        model_input = self.language_model.prepare_inputs_for_generation(
+            input_ids=input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            inputs_embeds=inputs_embeds,
+            cache_position=cache_position,
+            **kwargs,
+        )
+
+        # include audio information in model_input only when it is needed during prefilling
+        # audio_token_start_idx should always be relative to the current cache position
+        prefill_start_idx = 0 if cache_position is None else cache_position[0]
+        if (
+            audio_values is not None
+            and audio_token_start_idx is not None
+            and prefill_start_idx <= torch.max(audio_token_start_idx)
+        ):
+            model_input["audio_values"] = audio_values
+            model_input["audio_token_start_idx"] = (
+                audio_token_start_idx - prefill_start_idx
+            )
+            model_input["audio_token_len"] = audio_token_len
+            model_input["audio_len"] = audio_len
+
+        return model_input
+
+    @classmethod
+    def _create_multi_modal_projector(
+        cls, config: UltravoxConfig
+    ) -> "UltravoxProjector":
+        projector = UltravoxProjector(config)
+        projector.to(config.torch_dtype)
+        return projector
+
+    @classmethod
+    def _create_audio_tower(
+        cls, config: UltravoxConfig
+    ) -> Union[transformers.Wav2Vec2Model, "ModifiedWhisperEncoder"]:
+        if config.audio_model_id is not None:
+            if "whisper" in config.audio_model_id.lower():
+                audio_tower = ModifiedWhisperEncoder.from_pretrained(
+                    config.audio_model_id, torch_dtype=config.torch_dtype
+                )
+                audio_tower.init_latency_mask(
+                    config.audio_latency_block_size, dtype=config.torch_dtype
+                )
+            else:
+                assert config.audio_latency_block_size in (
+                    None,
+                    0,
+                ), "only whisper audio tower supports audio latency masking, got non-zero value for 'audio_latency_block_size'"
+                audio_tower = transformers.AutoModel.from_pretrained(
+                    config.audio_model_id, torch_dtype=config.torch_dtype
+                )
+        else:
+            if "whisper" in config.audio_config._name_or_path.lower():
+                audio_tower = ModifiedWhisperEncoder(config.audio_config)
+                audio_tower.init_latency_mask(
+                    config.audio_latency_block_size, dtype=config.torch_dtype
+                )
+            else:
+                assert config.audio_latency_block_size in (
+                    None,
+                    0,
+                ), "only whisper audio tower supports audio latency masking, got non-zero value for 'audio_latency_block_size'"
+                with transformers.modeling_utils.no_init_weights():
+                    # we only ever use from_config if the weights are retrained, hence initializing is not
+                    # required. This makes the model quite creation faster since init on CPU is quite slow.
+                    audio_tower = transformers.AutoModel.from_config(
+                        config.audio_config
+                    )
+
+        if isinstance(
+            audio_tower,
+            (transformers.Wav2Vec2BertModel, transformers.WhisperModel),
+        ):
+            # For these models we only need the encoder part
+            # Wav2Vec2BertModel -> Wav2Vec2BertEncoder
+            # WhisperModel -> WhisperEncoder
+            audio_tower = audio_tower.encoder
+
+        audio_tower = apply_lora(audio_tower, config.audio_model_lora_config)
+        return audio_tower
+
+    @classmethod
+    def _create_language_model(
+        cls, config: UltravoxConfig
+    ) -> transformers.LlamaForCausalLM:
+        if config.text_model_id is not None:
+            language_model = transformers.AutoModelForCausalLM.from_pretrained(
+                config.text_model_id,
+                attn_implementation=config._attn_implementation,
+                torch_dtype=config.torch_dtype,
+            )
+        else:
+            with transformers.modeling_utils.no_init_weights():
+                # we only ever use from_config if the weights are retrained, hence initializing is not
+                # required. This makes the model quite creation faster since init on CPU is quite slow.
+                language_model = transformers.AutoModelForCausalLM.from_config(
+                    config.text_config,
+                    attn_implementation=config._attn_implementation,
+                    torch_dtype=config.torch_dtype,
+                )
+
+        language_model = apply_lora(language_model, config.text_model_lora_config)
+        return language_model
+
+    def merge_and_unload(self):
+        if isinstance(self.language_model, peft.PeftModel):
+            self.language_model = self.language_model.merge_and_unload()
+            # no need to download base language model weights anymore, so we can remove the id
+            self.config.text_model_id = None
+            self.keep_params.update(
+                set(
+                    [
+                        f"language_model.{name}"
+                        for name, _ in self.language_model.named_parameters()
+                    ]
+                )
+            )
+
+        if isinstance(self.audio_tower, peft.PeftModel):
+            self.audio_tower = self.audio_tower.merge_and_unload()
+            # no need to download base audio model weights anymore, so we can remove the id
+            self.config.audio_model_id = None
+            self.keep_params.update(
+                set(
+                    [
+                        f"audio_tower.{name}"
+                        for name, _ in self.audio_tower.named_parameters()
+                    ]
+                )
+            )
+
+        for param in ["text_model_lora_config", "audio_model_lora_config"]:
+            if hasattr(self.config, param):
+                delattr(self.config, param)
+
+    def push_to_hub(self, *args, **kwargs):
+        self.merge_and_unload()
+        return super().push_to_hub(*args, **kwargs)
+
+    def diff_state_dict(
+        self, state_dict: Optional[Dict[str, Any]] = None
+    ) -> Dict[str, Any]:
+        if state_dict is None:
+            state_dict = super().state_dict()
+
+        named_params = dict(self.named_parameters())
+
+        state_dict = {
+            k: v
+            for k, v in state_dict.items()
+            if k in self.keep_params
+            or (k in named_params and named_params[k].requires_grad)
+        }
+
+        return state_dict
+
+    def save_pretrained(
+        self, *args, state_dict: Optional[Dict[str, Any]] = None, **kwargs
+    ):
+        state_dict = self.diff_state_dict(state_dict)
+
+        super().save_pretrained(*args, state_dict=state_dict, **kwargs)
+
+    def _pre_load_state_dict_hook(self, state_dict: Dict[str, Any], *args, **kwargs):
+        self.keep_params.update(set(state_dict.keys()))
+
+    def print_trainable_parameters(self):
+        """
+        Prints the number of trainable parameters in the model (reuses Peft model's method)
+        """
+        count_params = peft.peft_model.PeftModel.get_nb_trainable_parameters
+
+        trainable_params, all_param = count_params(self)
+
+        logging.info(
+            f"trainable params: {trainable_params:,d} || all params: {all_param:,d}"
+            f" || trainable%: {100 * trainable_params / all_param:.1f}%"
+        )
+
+        lm_trainable_params, lm_all_params = count_params(self.language_model)
+        audio_trainable_params, audio_all_params = count_params(self.audio_tower)
+
+        projector_trainable_params = (
+            trainable_params - lm_trainable_params - audio_trainable_params
+        )
+        projector_all_params = all_param - lm_all_params - audio_all_params
+
+        logging.info(
+            f"Trainable%:   "
+            f" LLM: {100 * lm_trainable_params / lm_all_params:.1f}%"
+            f" || Audio Encoder: {100 * audio_trainable_params / audio_all_params:.1f}%"
+            f" || Projector: {100 * projector_trainable_params / projector_all_params:.1f}%"
+        )
+
+
+# TODO: refactor common parts to a shared module
+def is_cache_empty(
+    past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]]
+) -> bool:
+    """
+    Check if the cache is empty.
+    """
+    if past_key_values is None:
+        return True
+    if isinstance(past_key_values, tuple):
+        return all(len(c) == 0 for c in past_key_values)
+    return past_key_values.get_seq_length() == 0
+
+
+def apply_lora(model: torch.nn.Module, lora_config: dict) -> torch.nn.Module:
+    """
+    Applies LoRA finetuning to the model. If the `r` parameter is set to 0, the model is frozen instead.
+    """
+    unfreeze_layers = lora_config.pop("unfreeze_layers", None)
+    lora_config = peft.LoraConfig(**lora_config or {})
+
+    if lora_config.r == 0:
+        # freeze the model entirely, except for the specified layers
+        for name, param in model.named_parameters():
+            if not unfreeze_layers or not any(
+                re.match(layer, name) for layer in unfreeze_layers
+            ):
+                param.requires_grad = False
+            else:
+                logging.info(f"Unfreezing layer: {name} with #{param.numel()} params")
+    else:
+        model = peft.get_peft_model(model, lora_config)
+
+    return model
+
+
+class StackAudioFrames(nn.Module):
+    """
+    Stack the audio embedding frames to reduce the sequence length by a factor of `stack_factor`.
+
+    The number of output frames will be `ceil(T / stack_factor) + 1` where `T` is the number of input frames.
+    NOTE: the extra +1 is intentional: in case the number of audio tokens are over-estimated by the processor,
+    we want to make sure `processor.audio_token_replacement` (i.e. EOS) doesn't get leaked into the middle of embeddings.
+    In most cases this extra padding will get removed in the model's forward function so it has no effect.
+    """
+
+    def __init__(self, stack_factor: int = 8):
+        super().__init__()
+        self.stack_factor = stack_factor
+
+    def forward(self, audio_embeds: torch.Tensor) -> torch.Tensor:
+        B, T, C = audio_embeds.shape
+        T_pad = (T + self.stack_factor - 1) // self.stack_factor * self.stack_factor
+        audio_embeds = F.pad(audio_embeds, (0, 0, 0, T_pad - T + self.stack_factor))
+        B, T, C = audio_embeds.shape
+        audio_embeds = audio_embeds.view(
+            B, T // self.stack_factor, C * self.stack_factor
+        )
+        return audio_embeds
+
+
+class RMSNorm(transformers.models.llama.modeling_llama.LlamaRMSNorm):
+    def __init__(self, hidden_size: int, init: float = 1, eps: float = 1e-6):
+        super().__init__(hidden_size=hidden_size, eps=eps)
+        self.weight.data.fill_(init)
+
+
+class SwiGLU(nn.Module):
+    def forward(self, x):
+        x, gate = x.chunk(2, dim=-1)
+        return F.silu(gate) * x
+
+
+class UltravoxProjector(nn.Module):
+    def __init__(self, config: UltravoxConfig):
+        super().__init__()
+        self.hidden_dim = config.hidden_size
+        self._pad_and_stack = StackAudioFrames(config.stack_factor)
+        dim_in = config.audio_config.hidden_size * config.stack_factor
+        self.ln_pre = RMSNorm(dim_in, init=config.norm_init)
+        self.linear_1 = nn.Linear(dim_in, self.hidden_dim, bias=False)
+        dim_mid = self.hidden_dim
+        self.act = transformers.activations.get_activation(config.projector_act)
+        dim_mid = dim_mid // 2 if config.projector_act == "swiglu" else dim_mid
+        dim_out = config.text_config.hidden_size
+        self.linear_2 = nn.Linear(dim_mid, dim_out, bias=False)
+
+        # Ultravox v0.4.1 and below uses layer_norm after the second linear layer,
+        # while v0.5.0 and above uses layer_norm after the first linear layer.
+        if config.projector_ln_mid:
+            self.ln_mid: nn.Module = RMSNorm(dim_mid, init=config.norm_init)
+            self.ln_post: nn.Module = nn.Identity()
+        else:
+            self.ln_mid = nn.Identity()
+            self.ln_post = RMSNorm(dim_out, init=config.norm_init)
+
+    def forward(self, audio_features: torch.Tensor) -> torch.Tensor:
+        audio_features = self._pad_and_stack(audio_features)
+        audio_features = self.ln_pre(audio_features)
+        hidden_states = self.linear_1(audio_features)
+        hidden_states = self.act(hidden_states)
+        hidden_states = self.ln_mid(hidden_states)
+        hidden_states = self.linear_2(hidden_states)
+        hidden_states = self.ln_post(hidden_states)
+        return hidden_states
+
+
+class ModifiedWhisperEncoder(
+    whisper.WhisperEncoder, transformers.modeling_utils.ModuleUtilsMixin
+):
+    """
+    Encoder portion of OpenAI's Whisper model.
+
+    This implementation is a slightly modified version of HF Transformers' Whisper Encoder, with only a few fixes:
+    1. base_model_prefix updated to allow for doing `.from_pretrained` directly on the encoder
+    2. allow less than 30 second of audio padding to be passed in:
+        - relaxed ValueError check for `input_features` length to be less than or equal to `expected_seq_length` instead of strictly equal
+        - embed_pos is now sliced to match the length of `inputs_embeds`
+
+    Original: https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/modeling_whisper.py
+    """
+
+    base_model_prefix = "model.encoder"
+    _no_split_modules = ["WhisperEncoderLayer"]
+
+    def __init__(self, config: transformers.WhisperConfig):
+        super().__init__(config)
+        self.config.is_decoder = False
+
+    def init_latency_mask(self, audio_latency_block_size: int, dtype: torch.dtype):
+        if audio_latency_block_size is None:
+            self.audio_streaming_mask = None
+            return
+
+        # maximum sequence length
+        max_seqlen = (
+            self.config.max_source_positions
+            * self.conv1.stride[0]
+            * self.conv2.stride[0]
+        )
+        assert (
+            max_seqlen > 0
+        ), f"maximum sequence length must be positive, got {max_seqlen}"
+        assert (
+            max_seqlen % audio_latency_block_size == 0
+        ), f"audio_latency_block_size {audio_latency_block_size} must divide {max_seqlen} evenly."
+        # Given the block size, we calculate number of blocks.
+        audio_latency_nblocks = max_seqlen // audio_latency_block_size
+        audio_streaming_mask = (
+            torch.tril(
+                torch.ones(audio_latency_nblocks, audio_latency_nblocks),
+                diagonal=0,
+            )
+            .repeat_interleave(audio_latency_block_size, dim=0)
+            .repeat_interleave(audio_latency_block_size, dim=1)
+        )
+        audio_streaming_mask = (1.0 - audio_streaming_mask) * torch.finfo(dtype).min
+        audio_streaming_mask = audio_streaming_mask[None, None, :, :]
+        self.register_buffer(
+            "audio_streaming_mask", audio_streaming_mask, persistent=False
+        )
+
+    def forward(
+        self,
+        input_features,
+        audio_len=None,
+        head_mask=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+    ):
+        expected_seq_length = (
+            self.config.max_source_positions
+            * self.conv1.stride[0]
+            * self.conv2.stride[0]
+        )
+        if input_features.shape[-1] > expected_seq_length:
+            raise ValueError(
+                f"Whisper expects the mel input features to be of length {expected_seq_length} or less, but found {input_features.shape[-1]}. Make sure to pad the input mel features to {expected_seq_length}."
+            )
+
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+        inputs_embeds = nn.functional.gelu(self.conv1(input_features))
+        inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds))
+
+        inputs_embeds = inputs_embeds.permute(0, 2, 1)
+        embed_pos = self.embed_positions.weight[: inputs_embeds.size(-2)]
+
+        hidden_states = inputs_embeds + embed_pos
+        hidden_states = nn.functional.dropout(
+            hidden_states, p=self.dropout, training=self.training
+        )
+
+        encoder_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+
+        # Create attention mask based on audio lengths to mask out padding tokens
+        # For each sample in batch:
+        # - Convert raw audio length to feature length after convolutions
+        # - Create boolean mask that is True for valid positions and False for padding
+        # - Convert to extended attention mask format expected by transformer layers
+        #   (1.0 for positions to attend to, large negative for positions to ignore)
+        # This masking ensures consistent behavior between training and inference
+        # by preventing the model from attending to padding tokens in both cases
+        attention_mask = None
+        if audio_len != None:
+            audio_feature_len = self._get_feat_extract_output_lengths(audio_len)
+            max_seq_len = hidden_states.shape[1]
+            attention_mask = torch.arange(max_seq_len, device=hidden_states.device)[
+                None, :
+            ].lt(audio_feature_len.view(-1, 1))
+            attention_mask = self.get_extended_attention_mask(
+                attention_mask,
+                None,
+                device=hidden_states.device,
+                dtype=hidden_states.dtype,
+            )
+
+        if self.audio_streaming_mask is not None:
+            seqlen = hidden_states.size(-2)
+            if attention_mask is not None:
+                attention_mask = torch.minimum(
+                    self.audio_streaming_mask[:, :, :seqlen, :seqlen], attention_mask
+                )  # merge
+            else:
+                attention_mask = self.audio_streaming_mask[:, :, :seqlen, :seqlen]
+            attention_mask = attention_mask.to(hidden_states.dtype)
+
+        # check if head_mask has a correct number of layers specified if desired
+        if head_mask is not None:
+            assert head_mask.size()[0] == (
+                len(self.layers)
+            ), f"The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.size()[0]}."
+
+        for idx, encoder_layer in enumerate(self.layers):
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
+            to_drop = False
+            if self.training:
+                dropout_probability = torch.rand([])
+                if dropout_probability < self.layerdrop:  # skip the layer
+                    to_drop = True
+
+            if to_drop:
+                layer_outputs = (None, None)
+            else:
+                if self.gradient_checkpointing and self.training:
+                    layer_outputs = self._gradient_checkpointing_func(
+                        encoder_layer.__call__,
+                        hidden_states,
+                        attention_mask,
+                        (head_mask[idx] if head_mask is not None else None),
+                        output_attentions,
+                    )
+                else:
+                    layer_outputs = encoder_layer(
+                        hidden_states,
+                        attention_mask,
+                        layer_head_mask=(
+                            head_mask[idx] if head_mask is not None else None
+                        ),
+                        output_attentions=output_attentions,
+                    )
+
+                hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+        hidden_states = self.layer_norm(hidden_states)
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [hidden_states, encoder_states, all_attentions]
+                if v is not None
+            )
+        return transformers.modeling_outputs.BaseModelOutput(
+            last_hidden_state=hidden_states,
+            hidden_states=encoder_states,
+            attentions=all_attentions,
+        )
+
+
+UltravoxConfig.register_for_auto_class()
+UltravoxModel.register_for_auto_class()
+
+transformers.AutoConfig.register("ultravox", UltravoxConfig)
+transformers.AutoModel.register(UltravoxConfig, UltravoxModel)
+
+transformers.activations.ACT2FN["swiglu"] = SwiGLU

checkpoint-7200/config.json

@@ -0,0 +1,44 @@
+{
+  "architectures": [
+    "UltravoxModel"
+  ],
+  "audio_latency_block_size": null,
+  "audio_model_id": "openai/whisper-large-v3-turbo",
+  "audio_model_lora_config": {
+    "lora_alpha": 8,
+    "r": 0,
+    "target_modules": [
+      "k_proj",
+      "q_proj",
+      "linear_k",
+      "linear_q"
+    ]
+  },
+  "auto_map": {
+    "AutoConfig": "ultravox_config.UltravoxConfig",
+    "AutoModel": "ultravox_model.UltravoxModel"
+  },
+  "hidden_size": 4096,
+  "ignore_index": -100,
+  "initializer_range": 0.02,
+  "model_type": "ultravox",
+  "norm_init": 0.4,
+  "pad_token_id": 2,
+  "projector_act": "swiglu",
+  "projector_ln_mid": false,
+  "stack_factor": 8,
+  "text_model_id": "HuggingFaceTB/SmolLM2-1.7B-Instruct",
+  "text_model_lora_config": {
+    "lora_alpha": 8,
+    "r": 0,
+    "target_modules": [
+      "k_proj",
+      "q_proj",
+      "linear_k",
+      "linear_q"
+    ]
+  },
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.47.0",
+  "vocab_size": 49152
+}

checkpoint-7200/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 2,
+  "transformers_version": "4.47.0"
+}

checkpoint-7200/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f21f8b45ee065ed6f618befeb1bfc6692a6bddd53677942d6441d37c9f2deb5
+size 92299736

checkpoint-7200/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1186377a7f21ea6b553cc9792f5a5dee90bb2759021e2938268a524806e4cef
+size 184602962