OpenLab-NLP
/

model-prototype

Model card Files Files and versions

Yuchan commited on Nov 24

Commit

2cdfb37

·

verified ·

1 Parent(s): 3153f7f

Update Mo.py

Files changed (1) hide show

Mo.py +2 -2

Mo.py CHANGED Viewed

@@ -69,7 +69,7 @@ vocab_size = sp.get_piece_size()
 print(f"✅ Vocabulary size: {vocab_size}")
 max_len = 256
-batch_size = 256
 def text_to_ids(text):
     return sp.encode(text, out_type=int)
@@ -99,7 +99,7 @@ def txt_stream(file_path):
             )
-LIMIT = 36757266
 dataset = tf.data.Dataset.from_generator(
     lambda: txt_stream(DATA_PATH),

 print(f"✅ Vocabulary size: {vocab_size}")
 max_len = 256
+batch_size = 128
 def text_to_ids(text):
     return sp.encode(text, out_type=int)
             )
+LIMIT = 36757266 // 2
 dataset = tf.data.Dataset.from_generator(
     lambda: txt_stream(DATA_PATH),