OpenLab-NLP
/

model-prototype

Model card Files Files and versions

Yuchan commited on 28 days ago

Commit

17d47d0

·

verified ·

1 Parent(s): 50f4d1a

Update Model.py

Files changed (1) hide show

Model.py +2 -1

Model.py CHANGED Viewed

@@ -213,10 +213,11 @@ class Block(layers.Layer):
         super().__init__()
         self.lou = LoU(d_model)
         self.glu = SwiGLU(d_model)
     def call(self, x):
         x = self.lou(x)
-        x = self.glu(x)
         return x
 class ReLM(tf.keras.Model):

         super().__init__()
         self.lou = LoU(d_model)
         self.glu = SwiGLU(d_model)
+        self.norm = layers.LayerNormalization(epsilon=1e-5, dtype='float32')
     def call(self, x):
         x = self.lou(x)
+        x = self.norm(self.glu(x)) + x
         return x
 class ReLM(tf.keras.Model):