Update Model.py

Model.py

@@ -223,7 +223,7 @@ class LoSoU(layers.Layer):
         score = g_q * g_k
 
         # 동적 alpha 계산: (B, L, d_model) -> (B, L, 1)
-        alpha_dynamic = self.alpha_linear(x_f32)  # (B, L, 1)
+        alpha_dynamic = self.alpha_linear(x_f32) * 0.8 + 0.1 # (B, L, 1)
         # 필요시 alpha_dynamic에 대한 후처리 (예: min/max 등) 가능
         # ex: alpha_dynamic = tf.clip_by_value(alpha_dynamic, 0.01, 0.99)
 
@@ -291,6 +291,7 @@ class ReLaM(tf.keras.Model):
         logits = tf.matmul(x, embedding_matrix, transpose_b=True)
         return tf.cast(logits, tf.float32)
 
+loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')
 
 def masked_loss(y_true, y_pred):
     loss = loss_fn(y_true, y_pred)