feat: simplified mon tokenizer in hf format, updated tags, resolve the legacy issue

convert_to_hf.py

@@ -9,7 +9,7 @@ convert_to_hf.py output files:
 - `tokenizer_config.json` - main config with modern `added_tokens_decoder` structure (not legacy)
 - `special_tokens_map.json` - special token definitions  
 - `generation_config.json` - generation parameters
-- `mon_tokenizer.model` - the sentencepiece model file
+- `tokenizer.model` - the sentencepiece model file
 - `README.md` - comprehensive model card
 - `.gitattributes` - git lfs configuration
 
@@ -40,7 +40,7 @@ class MonTokenizerConverter:
 
     def __init__(
         self,
-        model_file: str = "mon_tokenizer.model",
+        model_file: str = "tokenizer.model",
         meta_file: str = "mon_tokenizer.meta.json",
         output_dir: str = ".",
     ):
@@ -178,7 +178,7 @@ class MonTokenizerConverter:
             "tokenizer_class": "LlamaTokenizer",
             "unk_token": analysis["unk_token"],
             "use_default_system_prompt": False,
-            "vocab_file": "mon_tokenizer.model",
+            "vocab_file": "tokenizer.model",
             "vocab_size": analysis["vocab_size"]
         }
 
@@ -414,7 +414,7 @@ For questions or issues, please open an issue on the repository or contact the m
             analysis = self.analyze_sentencepiece_model()
 
             # Copy model file (use original name for compatibility)
-            model_dest = self.output_dir / "mon_tokenizer.model"
+            model_dest = self.output_dir / "tokenizer.model"
             if not model_dest.exists() or model_dest.resolve() != self.model_file.resolve():
                 logger.info("Copying SentencePiece model file")
                 shutil.copy2(self.model_file, model_dest)
@@ -469,8 +469,8 @@ def main():
     )
     parser.add_argument(
         "--model",
-        default="mon_tokenizer.model",
-        help="Path to SentencePiece model file (default: mon_tokenizer.model)",
+        default="tokenizer.model",
+        help="Path to SentencePiece model file (default: tokenizer.model)",
     )
     parser.add_argument(
         "--meta",

mon_tokenizer.meta.json

@@ -1,5 +1,5 @@
 {
-  "model_path": "mon_tokenizer.model",
+  "model_path": "tokenizer.model",
   "vocab_path": "mon_tokenizer.vocab",
   "lines_trained": 32412,
   "total_characters": 2453293,

tokenizer_config.json

@@ -47,6 +47,6 @@
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
   "use_default_system_prompt": false,
-  "vocab_file": "mon_tokenizer.model",
+  "vocab_file": "tokenizer.model",
   "vocab_size": 4000
 }

upload_to_hub.py

@@ -5,17 +5,24 @@ Upload Mon tokenizer to Hugging Face Hub.
 This script provides functionality to validate and upload the Mon language tokenizer
 to Hugging Face Hub with comprehensive validation and modern best practices.
 
- required files:
-- `tokenizer_config.json` 
-- `special_tokens_map.json`
-- `generation_config.json` 
-- `README.md`
-- `.gitattributes`
-- `mon_tokenizer.model` (auto-detects either `tokenizer.model` or `mon_tokenizer.model`)
+Required files:
+- `tokenizer_config.json` - Main tokenizer configuration
+- `special_tokens_map.json` - Special token mappings
+- `README.md` - Model documentation and usage instructions
+- `.gitattributes` - Git LFS configuration for large files
 
-Nothing missing - the script validates all files exist before upload and lists each file with size before uploading.
+Required tokenizer model files (at least one):
+- `tokenizer.json` - Fast tokenizer (recommended, HuggingFace Tokenizers)
+- `tokenizer.model` - SentencePiece model file (slow tokenizer)
+- `mon_tokenizer.model` - Custom named SentencePiece model
 
-The scripts now use modern conventions while maintaining backward compatibility through the `legacy: true` setting, which prevents the llama tokenizer warnings while using the latest transformers architecture.
+Optional but recommended files:
+- `generation_config.json` - Text generation configuration
+- `vocab.txt` - Vocabulary file for certain tokenizer types
+- `merges.txt` - BPE merge rules for certain tokenizer types
+
+The script validates all files exist before upload, supports both fast and slow tokenizers,
+and uses modern HuggingFace Hub conventions while maintaining backward compatibility.
 
 """
 
@@ -50,12 +57,20 @@ class TokenizerUploader:
         self.required_files = [
             "tokenizer_config.json",
             "special_tokens_map.json",
-            "generation_config.json",
             "README.md",
             ".gitattributes",
         ]
-        # Check for either tokenizer.model or mon_tokenizer.model
-        self.model_files = ["tokenizer.model", "mon_tokenizer.model"]
+        # Optional but recommended files
+        self.optional_files = [
+            "generation_config.json",
+            "vocab.txt",
+            "merges.txt",
+            "tokenizer.json",  # Fast tokenizer (becomes required if no .model file)
+            "added_tokens.json",  # Additional tokens
+            "preprocessor_config.json",  # Preprocessing configuration
+        ]
+        # Tokenizer model files - check for either fast or slow tokenizer
+        self.model_files = ["tokenizer.json", "tokenizer.model", "mon_tokenizer.model"]
 
     def validate_files(self) -> bool:
         """
@@ -80,31 +95,44 @@ class TokenizerUploader:
                 missing_files.append(file_name)
                 logger.error(f"✗ {file_name} (missing)")
 
-        # Check for model file (either name is acceptable)
+        # Check optional files
+        for file_name in self.optional_files:
+            file_path = self.directory / file_name
+            if file_path.exists():
+                size = file_path.stat().st_size
+                present_files.append((file_name, size))
+                logger.info(f"✓ {file_name} ({size:,} bytes) [optional]")
+
+        # Check for tokenizer model files - at least one must exist
         model_found = False
+        found_models = []
         for model_name in self.model_files:
             model_path = self.directory / model_name
             if model_path.exists():
                 size = model_path.stat().st_size
                 present_files.append((model_name, size))
+                found_models.append(model_name)
                 logger.info(f"✓ {model_name} ({size:,} bytes)")
                 model_found = True
-                break
         
         if not model_found:
-            missing_files.append("tokenizer.model or mon_tokenizer.model")
-            logger.error(f"✗ Model file missing (looked for: {', '.join(self.model_files)})")
+            missing_files.append("tokenizer model file (tokenizer.json, tokenizer.model, or mon_tokenizer.model)")
+            logger.error(f"✗ No tokenizer model file found (looked for: {', '.join(self.model_files)})")
+        else:
+            logger.info(f"✓ Found tokenizer model(s): {', '.join(found_models)}")
 
         if missing_files:
             logger.error(f"Missing required files: {', '.join(missing_files)}")
             return False
 
-        logger.info(f"✓ All {len(self.required_files)} required files present")
+        total_required = len(self.required_files) + 1  # +1 for model file
+        logger.info(f"✓ All {total_required} essential files present")
         return True
 
     def validate_tokenizer_functionality(self) -> bool:
         """
         Validate tokenizer functionality with comprehensive tests.
+        Supports both fast (tokenizer.json) and slow (tokenizer.model) tokenizers.
 
         Returns:
             bool: True if all tests pass, False otherwise
@@ -114,13 +142,24 @@ class TokenizerUploader:
         try:
             # Load tokenizer with explicit local files only
             abs_directory = str(self.directory.absolute())
+            
+            # Determine tokenizer type for better error handling
+            has_fast = (self.directory / "tokenizer.json").exists()
+            has_slow = any((self.directory / model).exists() for model in ["tokenizer.model", "mon_tokenizer.model"])
+            
+            if has_fast:
+                logger.info("Detected fast tokenizer (tokenizer.json)")
+            if has_slow:
+                logger.info("Detected slow tokenizer (*.model)")
+            
             tokenizer = AutoTokenizer.from_pretrained(
                 abs_directory, 
                 local_files_only=True,
                 trust_remote_code=False  # Security best practice
             )
 
-            logger.info(f"✓ Tokenizer loaded (vocab: {tokenizer.vocab_size:,})")
+            tokenizer_type = "fast" if tokenizer.is_fast else "slow"
+            logger.info(f"✓ {tokenizer_type.capitalize()} tokenizer loaded (vocab: {tokenizer.vocab_size:,})")
 
             # Comprehensive test cases for Mon language
             test_cases = [
@@ -288,20 +327,63 @@ class TokenizerUploader:
             # List files to upload
             upload_files = []
             ignore_patterns = [
+                # Python compilation artifacts
                 "*.pyc",
                 "__pycache__/",
+                "*.pyo",
+                
+                # Version control and development
                 ".git/",
+                ".gitignore",
                 ".venv/",
+                "venv/",
+                ".env",
+                ".env.*",
+                ".python-version",
+                
+                # Build and dependency files
                 "*.lock",
                 "uv.lock",
+                "Pipfile.lock",
+                "poetry.lock",
                 "pyproject.toml",
-                "datasets/",
-                "*.py",  # Don't upload Python scripts
+                "setup.py",
+                "setup.cfg",
+                "requirements.txt",
+                "requirements-dev.txt",
+                
+                # Development and testing files
                 "test_*",
+                "tests/",
+                "*_test.py",
                 "sample_*",
+                "example_*",
+                "demo_*",
+                
+                # Build and conversion scripts
                 "convert_*",
-                "upload_*",
-                "*.meta.json"
+                "upload_*", 
+                "build_*",
+                "*.py",  # Don't upload Python scripts
+                
+                # Dataset and training artifacts
+                "datasets/",
+                "data/",
+                "checkpoints/",
+                "logs/",
+                "wandb/",
+                
+                # Temporary and cache files
+                "*.tmp",
+                "*.temp",
+                ".cache/",
+                "*.meta.json",
+                "*.backup",
+                
+                # OS specific files
+                ".DS_Store",
+                "Thumbs.db",
+                "desktop.ini"
             ]
 
             logger.info("Files to be uploaded:")