Training in progress - step 500

asr_config.py

@@ -26,6 +26,7 @@ class ASRConfig(transformers.PretrainedConfig):
         projector_num_layers: int = 2,  # Number of layers (for residual projector)
         projector_dropout: float = 0.05,  # Dropout rate for projector layers
         projector_input_noise: float = 0.02,  # Input noise for projector
+        label_smoothing: float = 0.0,  # Label smoothing for cross-entropy loss
         inference_diversity_penalty: float = 0.0,
         inference_warmup_tokens: int = 10,
         max_new_tokens: Optional[int] = None,
@@ -72,6 +73,7 @@ class ASRConfig(transformers.PretrainedConfig):
         self.projector_num_layers = projector_num_layers
         self.projector_dropout = projector_dropout
         self.projector_input_noise = projector_input_noise
+        self.label_smoothing = label_smoothing
         self.inference_diversity_penalty = inference_diversity_penalty
         self.inference_warmup_tokens = inference_warmup_tokens
         if "audio_config" not in kwargs:

asr_modeling.py

@@ -118,10 +118,6 @@ class ASRModel(PreTrainedModel):
             "low_cpu_mem_usage": True,
             "dtype": dtype,
         }
-        # Only use device_map="auto" when NOT loading from pretrained
-        # (avoids meta tensor conflicts during from_pretrained)
-        if not cls._is_loading_from_pretrained:
-            encoder_kwargs["device_map"] = "auto"
 
         if "whisper" in config.audio_model_id.lower():
             from transformers import WhisperModel
@@ -146,9 +142,6 @@ class ASRModel(PreTrainedModel):
             "low_cpu_mem_usage": True,
             "dtype": dtype,
         }
-        # Only use device_map="auto" when NOT loading from pretrained
-        if not cls._is_loading_from_pretrained:
-            decoder_kwargs["device_map"] = "auto"
 
         decoder = AutoModelForCausalLM.from_pretrained(config.text_model_id, **decoder_kwargs)
         decoder.config.use_cache = getattr(config, "use_cache", True)
@@ -393,6 +386,7 @@ class ASRModel(PreTrainedModel):
                 shift_logits.view(-1, shift_logits.size(-1)),
                 shift_labels.view(-1),
                 ignore_index=-100,
+                label_smoothing=getattr(self.config, "label_smoothing", 0.0),
             )
 
         return CausalLMOutputWithPast(

config.json

@@ -68,6 +68,7 @@
   "encoder_dim": 1280,
   "inference_diversity_penalty": 0.0,
   "inference_warmup_tokens": 10,
+  "label_smoothing": 0.1,
   "llm_dim": 2048,
   "max_new_tokens": 128,
   "min_new_tokens": 1,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d5cdacf5b89d706d6f0e5faffce196310de913bd0a577f4bdbe7b92335271e59
+oid sha256:3a25deecc1f7a0eef0322e88451ff27f6eb9ade7e853e21df8eb8afee152b736
 size 144762160