Update Model.py

Model.py

@@ -191,10 +191,12 @@ class Lo(layers.Layer):
         super().__init__()
         self.d = layers.Dense(256, activation='silu')
         self.w = layers.Dense(d_model)
+        self.norm = layers.LayerNormalization(epsilon=1e-5, dtype='float32')
+
     def call(self, x):
         p = self.d(x)
         p = self.w(p)
-        return p + x
+        return self.norm(p) + x
 
 class Block(layers.Layer):
     def __init__(self, d_model):
@@ -256,7 +258,7 @@ model = ReLM(
     vocab_size=vocab_size,
     max_seq_len=max_len,
     d_model=700,
-    n_layers=2
+    n_layers=16
 )
 
 # 옵티마이저 설정