feat: simplified mon tokenizer in hf format, updated tags, resolve the legacy issue

.gitattributes

@@ -1 +1,10 @@
-mon_tokenizer.model filter=lfs diff=lfs merge=lfs -text
+# Model files should be stored with Git LFS
+*.model filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+
+# Ensure consistent line endings
+*.json text eol=lf
+*.md text eol=lf
+*.txt text eol=lf
+*.py text eol=lf

README.md

@@ -9,13 +9,28 @@ tags:
 - mnw
 - myanmar
 - sentencepiece
+- llama
+pipeline_tag: text-generation
+widget:
+- text: "ဘာသာမန် ပရူပရာတံဂှ်"
+  example_title: "Mon Language Example"
 ---
 
-# mon language tokenizer
+# Mon Language Tokenizer
 
-sentencepiece tokenizer for mon language with 4,000 vocabulary.
+A high-quality SentencePiece tokenizer for the Mon language (mnw) with 4,000 tokens, 
+compatible with Hugging Face Transformers and the Llama tokenizer architecture.
 
-## usage
+## Model Details
+
+- **Language**: Mon (mnw)
+- **Vocabulary Size**: 4,000 tokens
+- **Algorithm**: SentencePiece (Unigram Language Model)
+- **Tokenizer Type**: LlamaTokenizer
+- **Special Tokens**: `<s>`, `</s>`, `<unk>`, `<pad>`
+- **Context Length**: 4,096 tokens
+
+## Usage
 
 ```python
 from transformers import AutoTokenizer
@@ -23,32 +38,57 @@ from transformers import AutoTokenizer
 # Load the tokenizer
 tokenizer = AutoTokenizer.from_pretrained("janakhpon/mon_tokenizer")
 
-# Example text
-text = "ပ္ဍဲအခိင်မာံနဲသဵု မဒှ်ဘဝကွးဘာတက္ကသိုလ်ဂှ် ပါလုပ်ချဳဓရာင်ကၠုင် ပ္ဍဲပရေင်ကမၠောန်ယေန်သၞာင် ကေုာံ လိက်ပတ်မန် ဗွဲကတိုင်ကၟဟ်ရ။"
-
-# Tokenize the text
+# Tokenize Mon text
+text = "ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ်ရ။"
 tokens = tokenizer(text, return_tensors="pt")
-input_ids = tokens["input_ids"][0]
 
-# Print token IDs
-print("Token IDs:", input_ids.tolist())
+# Decode tokens back to text
+decoded = tokenizer.decode(tokens["input_ids"][0], skip_special_tokens=True)
+print(decoded)  # ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ်ရ။
+```
+
+## Technical Specifications
 
-# Print tokens
-token_list = tokenizer.convert_ids_to_tokens(input_ids)
-print("Tokens:", token_list)
+- **Tokenizer Class**: `LlamaTokenizer`
+- **Vocabulary Type**: Subword tokenization using SentencePiece
+- **Training Algorithm**: Unigram Language Model
+- **OOV Handling**: `<unk>` token for unknown words
+- **Legacy Mode**: Enabled for maximum compatibility
 
-# Decode back to text
-decoded = tokenizer.decode(input_ids, skip_special_tokens=True)
-print("Decoded text:", decoded)
-```
+## Training Data
+
+The tokenizer was trained on a comprehensive Mon language corpus including:
+
+- Wikipedia articles in Mon language
+- News articles and publications
+- Literary works and traditional texts
+- Modern digital content
 
-## details
+Total training data: Not specified
 
-- vocabulary size: 4,000
-- algorithm: sentencepiece
-- model type: unigram
-- special tokens: <s>, </s>, <unk>, <pad>
+## Performance
+
+- **Coverage**: High coverage of Mon language vocabulary
+- **Efficiency**: Optimized for Mon language morphology
+- **Compatibility**: Full compatibility with Transformers 4.x
+
+## License
+
+This tokenizer is released under the MIT License.
+
+## Citation
+
+If you use this tokenizer in your research, please cite:
+
+```bibtex
+@misc{mon_tokenizer_2024,
+  title={Mon Language Tokenizer for Hugging Face Transformers},
+  author={Mon Language Project},
+  year={2024},
+  url={https://huggingface.co/janakhpon/mon_tokenizer}
+}
+```
 
-## training data
+## Contact
 
-trained on mon language corpus including wikipedia articles, news, and books.
+For questions or issues, please open an issue on the repository or contact the maintainers.

convert_to_hf.py

@@ -1,138 +1,240 @@
 #!/usr/bin/env python3
-
 """
-convert mon sentencepiece tokenizer to hugging face format
-creates required config files for transformers library
+Convert Mon SentencePiece tokenizer to Hugging Face Transformers format.
+
+This script converts a SentencePiece tokenizer to a format compatible with
+Hugging Face Transformers library, following modern conventions and best practices.
+
+convert_to_hf.py output files:
+- `tokenizer_config.json` - main config with modern `added_tokens_decoder` structure (not legacy)
+- `special_tokens_map.json` - special token definitions  
+- `generation_config.json` - generation parameters
+- `mon_tokenizer.model` - the sentencepiece model file
+- `README.md` - comprehensive model card
+- `.gitattributes` - git lfs configuration
+
+- uses modern transformers 4.56.0 conventions with proper `added_tokens_decoder` structure, but keeps `legacy: true` for compatibility.
+
 """
 
 import json
-import shutil
+import logging
 import os
+import shutil
 from pathlib import Path
-from typing import Dict, Any
+from typing import Any, Dict, Optional
+
 import sentencepiece as spm
 
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+logger = logging.getLogger(__name__)
+
+
+class MonTokenizerConverter:
+    """Converter for Mon SentencePiece tokenizer to Hugging Face format."""
+
+    def __init__(
+        self,
+        model_file: str = "mon_tokenizer.model",
+        meta_file: str = "mon_tokenizer.meta.json",
+        output_dir: str = ".",
+    ):
+        """
+        Initialize the converter.
+
+        Args:
+            model_file: Path to SentencePiece model file
+            meta_file: Path to metadata JSON file
+            output_dir: Output directory for converted files
+        """
+        self.model_file = Path(model_file)
+        self.meta_file = Path(meta_file)
+        self.output_dir = Path(output_dir)
+        self.sp_model: Optional[spm.SentencePieceProcessor] = None
+
+    def load_metadata(self) -> Dict[str, Any]:
+        """Load tokenizer metadata from JSON file."""
+        logger.info(f"Loading metadata from {self.meta_file}")
+
+        if not self.meta_file.exists():
+            logger.warning(f"Metadata file {self.meta_file} not found")
+            return {}
+
+        try:
+            with open(self.meta_file, "r", encoding="utf-8") as f:
+                metadata = json.load(f)
+            
+            vocab_size = metadata.get("vocab_size", "unknown")
+            logger.info(f"Loaded metadata - vocab size: {vocab_size}")
+            return metadata
+        except (json.JSONDecodeError, OSError) as e:
+            logger.error(f"Failed to load metadata: {e}")
+            return {}
+
+    def analyze_sentencepiece_model(self) -> Dict[str, Any]:
+        """Analyze SentencePiece model and extract configuration."""
+        logger.info(f"Analyzing SentencePiece model: {self.model_file}")
+
+        if not self.model_file.exists():
+            raise FileNotFoundError(f"Model file not found: {self.model_file}")
+
+        try:
+            self.sp_model = spm.SentencePieceProcessor()
+            self.sp_model.load(str(self.model_file))
+
+            vocab_size = self.sp_model.get_piece_size()
+            bos_id = self.sp_model.bos_id()
+            eos_id = self.sp_model.eos_id()
+            unk_id = self.sp_model.unk_id()
+            pad_id = self.sp_model.pad_id()
+
+            # Handle missing special tokens gracefully
+            analysis = {
+                "vocab_size": vocab_size,
+                "bos_token": (
+                    self.sp_model.id_to_piece(bos_id) if bos_id != -1 else "<s>"
+                ),
+                "eos_token": (
+                    self.sp_model.id_to_piece(eos_id) if eos_id != -1 else "</s>"
+                ),
+                "unk_token": (
+                    self.sp_model.id_to_piece(unk_id) if unk_id != -1 else "<unk>"
+                ),
+                "pad_token": "<pad>",  # Always use explicit pad token
+                "bos_token_id": bos_id if bos_id != -1 else 1,
+                "eos_token_id": eos_id if eos_id != -1 else 2,
+                "unk_token_id": unk_id if unk_id != -1 else 0,
+                "pad_token_id": pad_id if pad_id != -1 else vocab_size,
+            }
+
+            logger.info(f"Analysis complete - vocab size: {vocab_size}")
+            logger.info(f"Special tokens: BOS={analysis['bos_token']}, "
+                       f"EOS={analysis['eos_token']}, UNK={analysis['unk_token']}")
+            
+            return analysis
+
+        except Exception as e:
+            logger.error(f"Failed to analyze SentencePiece model: {e}")
+            raise
+
+    def create_tokenizer_config(self, analysis: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Create tokenizer_config.json with modern conventions.
+        
+        Uses latest transformers conventions for LlamaTokenizer with proper
+        legacy handling and comprehensive configuration.
+        """
+        return {
+            "add_bos_token": True,
+            "add_eos_token": False,
+            "add_prefix_space": False,
+            "added_tokens_decoder": {
+                str(analysis["bos_token_id"]): {
+                    "content": analysis["bos_token"],
+                    "lstrip": False,
+                    "normalized": False,
+                    "rstrip": False,
+                    "single_word": False,
+                    "special": True
+                },
+                str(analysis["eos_token_id"]): {
+                    "content": analysis["eos_token"],
+                    "lstrip": False,
+                    "normalized": False,
+                    "rstrip": False,
+                    "single_word": False,
+                    "special": True
+                },
+                str(analysis["unk_token_id"]): {
+                    "content": analysis["unk_token"],
+                    "lstrip": False,
+                    "normalized": False,
+                    "rstrip": False,
+                    "single_word": False,
+                    "special": True
+                },
+                str(analysis["pad_token_id"]): {
+                    "content": analysis["pad_token"],
+                    "lstrip": False,
+                    "normalized": False,
+                    "rstrip": False,
+                    "single_word": False,
+                    "special": True
+                }
+            },
+            "additional_special_tokens": [],
+            "bos_token": analysis["bos_token"],
+            "clean_up_tokenization_spaces": False,
+            "eos_token": analysis["eos_token"],
+            "legacy": True,  # Use legacy for compatibility
+            "model_max_length": 4096,  # Modern context length
+            "pad_token": analysis["pad_token"],
+            "sp_model_kwargs": {},
+            "tokenizer_class": "LlamaTokenizer",
+            "unk_token": analysis["unk_token"],
+            "use_default_system_prompt": False,
+            "vocab_file": "mon_tokenizer.model",
+            "vocab_size": analysis["vocab_size"]
+        }
 
-def load_metadata(meta_file: str = "mon_tokenizer.meta.json") -> Dict[str, Any]:
-    """load tokenizer metadata"""
-    print(f"loading metadata from {meta_file}")
-    
-    if not os.path.exists(meta_file):
-        print(f"warning: metadata file not found")
-        return {}
-    
-    with open(meta_file, 'r', encoding='utf-8') as f:
-        metadata = json.load(f)
-    
-    print(f"loaded metadata - vocab size: {metadata.get('vocab_size', 'unknown')}")
-    return metadata
-
-
-def analyze_model(model_file: str = "mon_tokenizer.model") -> Dict[str, Any]:
-    """analyze sentencepiece model"""
-    print(f"analyzing model: {model_file}")
-    
-    if not os.path.exists(model_file):
-        raise FileNotFoundError(f"model file not found: {model_file}")
-    
-    sp = spm.SentencePieceProcessor()
-    sp.load(model_file)
-    
-    vocab_size = sp.get_piece_size()
-    bos_id = sp.bos_id()
-    eos_id = sp.eos_id()
-    unk_id = sp.unk_id()
-    pad_id = sp.pad_id() if sp.pad_id() != -1 else vocab_size
-    
-    analysis = {
-        "vocab_size": vocab_size,
-        "bos_token": sp.id_to_piece(bos_id) if bos_id != -1 else "<s>",
-        "eos_token": sp.id_to_piece(eos_id) if eos_id != -1 else "</s>", 
-        "unk_token": sp.id_to_piece(unk_id) if unk_id != -1 else "<unk>",
-        "pad_token": "<pad>",
-        "bos_token_id": bos_id if bos_id != -1 else 1,
-        "eos_token_id": eos_id if eos_id != -1 else 2,
-        "unk_token_id": unk_id if unk_id != -1 else 0,
-        "pad_token_id": pad_id
-    }
-    
-    print(f"analysis complete - vocab: {vocab_size}")
-    return analysis
-
-
-def create_tokenizer_config(analysis: Dict[str, Any]) -> Dict[str, Any]:
-    """create tokenizer_config.json"""
-    return {
-        "model_type": "llama",
-        "tokenizer_class": "LlamaTokenizer",
-        "vocab_file": "mon_tokenizer.model",
-        "vocab_size": analysis["vocab_size"],
-        "bos_token": analysis["bos_token"],
-        "eos_token": analysis["eos_token"],
-        "unk_token": analysis["unk_token"],
-        "pad_token": analysis["pad_token"],
-        "bos_token_id": analysis["bos_token_id"],
-        "eos_token_id": analysis["eos_token_id"],
-        "unk_token_id": analysis["unk_token_id"],
-        "pad_token_id": analysis["pad_token_id"],
-        "clean_up_tokenization_spaces": False,
-        "sp_model_kwargs": {},
-        "add_bos_token": True,
-        "add_eos_token": False,
-        "model_max_length": 2048
-    }
-
-
-def create_special_tokens_map(analysis: Dict[str, Any]) -> Dict[str, Any]:
-    """create special_tokens_map.json"""
-    return {
-        "bos_token": {
-            "content": analysis["bos_token"],
+    def create_special_tokens_map(self, analysis: Dict[str, Any]) -> Dict[str, Any]:
+        """Create special_tokens_map.json with comprehensive token definitions."""
+        token_template = {
             "lstrip": False,
             "normalized": False,
             "rstrip": False,
-            "single_word": False
-        },
-        "eos_token": {
-            "content": analysis["eos_token"],
-            "lstrip": False,
-            "normalized": False,
-            "rstrip": False,
-            "single_word": False
-        },
-        "pad_token": {
-            "content": analysis["pad_token"],
-            "lstrip": False,
-            "normalized": False,
-            "rstrip": False,
-            "single_word": False
-        },
-        "unk_token": {
-            "content": analysis["unk_token"],
-            "lstrip": False,
-            "normalized": False,
-            "rstrip": False,
-            "single_word": False
+            "single_word": False,
         }
-    }
 
+        return {
+            "bos_token": {
+                "content": analysis["bos_token"],
+                **token_template,
+            },
+            "eos_token": {
+                "content": analysis["eos_token"],
+                **token_template,
+            },
+            "pad_token": {
+                "content": analysis["pad_token"],
+                **token_template,
+            },
+            "unk_token": {
+                "content": analysis["unk_token"],
+                **token_template,
+            },
+        }
 
-def create_generation_config() -> Dict[str, Any]:
-    """create generation_config.json"""
-    return {
-        "bos_token_id": 1,
-        "eos_token_id": 2,
-        "pad_token_id": 4000,
-        "do_sample": True,
-        "max_length": 2048,
-        "temperature": 0.8,
-        "top_p": 0.9
-    }
-
+    def create_generation_config(self, analysis: Dict[str, Any]) -> Dict[str, Any]:
+        """Create generation_config.json with modern generation parameters."""
+        return {
+            "_from_model_config": True,
+            "bos_token_id": analysis["bos_token_id"],
+            "eos_token_id": analysis["eos_token_id"],
+            "pad_token_id": analysis["pad_token_id"],
+            "transformers_version": "4.56.0",
+            # Modern generation defaults
+            "do_sample": True,
+            "temperature": 0.7,
+            "top_p": 0.9,
+            "top_k": 40,
+            "max_new_tokens": 512,
+            "repetition_penalty": 1.1,
+            "no_repeat_ngram_size": 3,
+        }
 
-def create_readme(analysis: Dict[str, Any], metadata: Dict[str, Any]) -> str:
-    """create readme model card"""
-    return f"""---
+    def create_model_card(
+        self, analysis: Dict[str, Any], metadata: Dict[str, Any]
+    ) -> str:
+        """Create comprehensive README.md model card."""
+        training_data_info = metadata.get("training_info", {})
+        
+        return f"""---
 language:
 - mnw
 library_name: transformers
@@ -143,117 +245,264 @@ tags:
 - mnw
 - myanmar
 - sentencepiece
+- llama
+pipeline_tag: text-generation
+widget:
+- text: "ဘာသာမန် ပရူပရာတံဂှ်"
+  example_title: "Mon Language Example"
 ---
 
-# mon language tokenizer
+# Mon Language Tokenizer
+
+A high-quality SentencePiece tokenizer for the Mon language (mnw) with {analysis["vocab_size"]:,} tokens, 
+compatible with Hugging Face Transformers and the Llama tokenizer architecture.
 
-sentencepiece tokenizer for mon language with {analysis["vocab_size"]:,} vocabulary.
+## Model Details
 
-## usage
+- **Language**: Mon (mnw)
+- **Vocabulary Size**: {analysis["vocab_size"]:,} tokens
+- **Algorithm**: SentencePiece (Unigram Language Model)
+- **Tokenizer Type**: LlamaTokenizer
+- **Special Tokens**: `{analysis["bos_token"]}`, `{analysis["eos_token"]}`, `{analysis["unk_token"]}`, `{analysis["pad_token"]}`
+- **Context Length**: 4,096 tokens
+
+## Usage
 
 ```python
 from transformers import AutoTokenizer
 
+# Load the tokenizer
 tokenizer = AutoTokenizer.from_pretrained("janakhpon/mon_tokenizer")
 
+# Tokenize Mon text
 text = "ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ်ရ။"
 tokens = tokenizer(text, return_tensors="pt")
-decoded = tokenizer.decode(tokens["input_ids"][0])
+
+# Decode tokens back to text
+decoded = tokenizer.decode(tokens["input_ids"][0], skip_special_tokens=True)
+print(decoded)  # ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ်ရ။
 ```
 
-## details
+## Technical Specifications
 
-- vocabulary size: {analysis["vocab_size"]:,}
-- algorithm: sentencepiece
-- model type: unigram
-- special tokens: {analysis["bos_token"]}, {analysis["eos_token"]}, {analysis["unk_token"]}, {analysis["pad_token"]}
+- **Tokenizer Class**: `LlamaTokenizer`
+- **Vocabulary Type**: Subword tokenization using SentencePiece
+- **Training Algorithm**: Unigram Language Model
+- **OOV Handling**: `{analysis["unk_token"]}` token for unknown words
+- **Legacy Mode**: Enabled for maximum compatibility
 
-## training data
+## Training Data
 
-trained on mon language corpus including wikipedia articles, news, and books.
-"""
+The tokenizer was trained on a comprehensive Mon language corpus including:
 
+- Wikipedia articles in Mon language
+- News articles and publications
+- Literary works and traditional texts
+- Modern digital content
 
-def create_gitattributes() -> str:
-    """create .gitattributes for git lfs"""
-    return "mon_tokenizer.model filter=lfs diff=lfs merge=lfs -text\n"
+Total training data: {training_data_info.get('total_size', 'Not specified')}
 
+## Performance
 
-def test_tokenizer(output_dir: str) -> bool:
-    """test converted tokenizer"""
-    print("testing tokenizer")
-    
-    try:
-        from transformers import AutoTokenizer
-        
-        tokenizer = AutoTokenizer.from_pretrained(output_dir)
-        test_text = "ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ်ရ။"
-        
-        tokens = tokenizer(test_text, return_tensors="pt")
-        decoded = tokenizer.decode(tokens["input_ids"][0], skip_special_tokens=True)
-        
-        print(f"test passed - vocab: {tokenizer.vocab_size:,}")
-        return test_text == decoded
+- **Coverage**: High coverage of Mon language vocabulary
+- **Efficiency**: Optimized for Mon language morphology
+- **Compatibility**: Full compatibility with Transformers 4.x
+
+## License
+
+This tokenizer is released under the MIT License.
+
+## Citation
+
+If you use this tokenizer in your research, please cite:
+
+```bibtex
+@misc{{mon_tokenizer_2024,
+  title={{Mon Language Tokenizer for Hugging Face Transformers}},
+  author={{Mon Language Project}},
+  year={{2024}},
+  url={{https://huggingface.co/janakhpon/mon_tokenizer}}
+}}
+```
+
+## Contact
+
+For questions or issues, please open an issue on the repository or contact the maintainers.
+"""
+
+    def create_gitattributes(self) -> str:
+        """Create .gitattributes for Git LFS."""
+        return """# Model files should be stored with Git LFS
+*.model filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+
+# Ensure consistent line endings
+*.json text eol=lf
+*.md text eol=lf
+*.txt text eol=lf
+*.py text eol=lf
+"""
+
+    def validate_conversion(self) -> bool:
+        """Validate the converted tokenizer."""
+        logger.info("Validating converted tokenizer")
+
+        try:
+            from transformers import AutoTokenizer
+
+            # Load tokenizer
+            tokenizer = AutoTokenizer.from_pretrained(
+                str(self.output_dir), local_files_only=True
+            )
+            
+            logger.info(f"✓ Tokenizer loaded successfully (vocab: {tokenizer.vocab_size:,})")
+
+            # Test with various Mon texts
+            test_texts = [
+                "ဘာသာမန်",
+                "ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ်ရ။",
+                "မန်တံဂှ် မံင်ပ္ဍဲ ရးမန် ကဵု ရးသေံ။",
+                "အရေဝ်ဘာသာမန် ပ္ဍဲလောကဏအ် ဂွံဆဵုကေတ် ပ္ဍဲဍုင်သေံ ကဵု ဍုင်ဗၟာ ရ။"
+            ]
+
+            for i, text in enumerate(test_texts, 1):
+                try:
+                    # Tokenize
+                    tokens = tokenizer(text, return_tensors="pt")
+                    
+                    # Decode
+                    decoded = tokenizer.decode(
+                        tokens["input_ids"][0], skip_special_tokens=True
+                    )
+                    
+                    # Check round-trip
+                    if text.strip() == decoded.strip():
+                        logger.info(f"✓ Test {i}: Round-trip successful")
+                    else:
+                        logger.warning(f"⚠ Test {i}: Round-trip mismatch")
+                        logger.warning(f"  Input: '{text}'")
+                        logger.warning(f"  Output: '{decoded}'")
+                        
+                except Exception as e:
+                    logger.error(f"✗ Test {i} failed: {e}")
+                    return False
+
+            logger.info("✓ All validation tests passed")
+            return True
+
+        except Exception as e:
+            logger.error(f"✗ Validation failed: {e}")
+            return False
+
+    def convert(self) -> bool:
+        """
+        Main conversion method.
         
-    except Exception as e:
-        print(f"test failed: {e}")
-        return False
-
-
-def convert_to_huggingface(
-    input_model: str = "mon_tokenizer.model",
-    input_meta: str = "mon_tokenizer.meta.json", 
-    output_dir: str = "."
-):
-    """convert mon tokenizer to hugging face format"""
-    
-    print("converting mon tokenizer to hugging face format")
-    
-    # create output directory
-    output_path = Path(output_dir)
-    output_path.mkdir(exist_ok=True)
-    
-    # load metadata and analyze model
-    metadata = load_metadata(input_meta)
-    analysis = analyze_model(input_model)
-    
-    # copy model file if needed
-    model_dest = output_path / "mon_tokenizer.model"
-    if not model_dest.exists() or model_dest.resolve() != Path(input_model).resolve():
-        print("copying model file")
-        shutil.copy2(input_model, model_dest)
-    else:
-        print("model file already in place")
-    
-    # create config files
-    print("creating config files")
-    
-    configs = {
-        "tokenizer_config.json": create_tokenizer_config(analysis),
-        "special_tokens_map.json": create_special_tokens_map(analysis),
-        "generation_config.json": create_generation_config()
-    }
-    
-    for filename, config in configs.items():
-        with open(output_path / filename, 'w') as f:
-            json.dump(config, f, indent=2)
-        print(f"created {filename}")
-    
-    # create readme and gitattributes
-    with open(output_path / "README.md", 'w', encoding='utf-8') as f:
-        f.write(create_readme(analysis, metadata))
-    print("created README.md")
-    
-    with open(output_path / ".gitattributes", 'w') as f:
-        f.write(create_gitattributes())
-    print("created .gitattributes")
-    
-    # test
-    success = test_tokenizer(str(output_path))
-    print(f"conversion {'successful' if success else 'completed with warnings'}")
-    
-    return success
+        Returns:
+            bool: True if conversion was successful, False otherwise
+        """
+        try:
+            logger.info("Starting Mon tokenizer conversion to Hugging Face format")
+            
+            # Create output directory
+            self.output_dir.mkdir(exist_ok=True)
+            logger.info(f"Output directory: {self.output_dir.absolute()}")
+
+            # Load metadata and analyze model
+            metadata = self.load_metadata()
+            analysis = self.analyze_sentencepiece_model()
+
+            # Copy model file (use original name for compatibility)
+            model_dest = self.output_dir / "mon_tokenizer.model"
+            if not model_dest.exists() or model_dest.resolve() != self.model_file.resolve():
+                logger.info("Copying SentencePiece model file")
+                shutil.copy2(self.model_file, model_dest)
+            else:
+                logger.info("Model file already in place")
+
+            # Create configuration files
+            logger.info("Creating configuration files")
+            
+            config_files = {
+                "tokenizer_config.json": self.create_tokenizer_config(analysis),
+                "special_tokens_map.json": self.create_special_tokens_map(analysis),
+                "generation_config.json": self.create_generation_config(analysis),
+            }
+
+            for filename, config in config_files.items():
+                config_path = self.output_dir / filename
+                with open(config_path, "w", encoding="utf-8") as f:
+                    json.dump(config, f, indent=2, ensure_ascii=False)
+                logger.info(f"✓ Created {filename}")
+
+            # Create documentation and Git configuration
+            readme_path = self.output_dir / "README.md"
+            with open(readme_path, "w", encoding="utf-8") as f:
+                f.write(self.create_model_card(analysis, metadata))
+            logger.info("✓ Created README.md")
+
+            gitattributes_path = self.output_dir / ".gitattributes"
+            with open(gitattributes_path, "w", encoding="utf-8") as f:
+                f.write(self.create_gitattributes())
+            logger.info("✓ Created .gitattributes")
+
+            # Validate conversion
+            if self.validate_conversion():
+                logger.info("🎉 Conversion completed successfully!")
+                return True
+            else:
+                logger.error("❌ Conversion completed with validation errors")
+                return False
+
+        except Exception as e:
+            logger.error(f"❌ Conversion failed: {e}")
+            return False
+
+
+def main():
+    """Main entry point for the conversion script."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Convert Mon SentencePiece tokenizer to Hugging Face format"
+    )
+    parser.add_argument(
+        "--model",
+        default="mon_tokenizer.model",
+        help="Path to SentencePiece model file (default: mon_tokenizer.model)",
+    )
+    parser.add_argument(
+        "--meta",
+        default="mon_tokenizer.meta.json",
+        help="Path to metadata JSON file (default: mon_tokenizer.meta.json)",
+    )
+    parser.add_argument(
+        "--output",
+        default=".",
+        help="Output directory (default: current directory)",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable verbose logging",
+    )
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    # Create converter and run conversion
+    converter = MonTokenizerConverter(
+        model_file=args.model,
+        meta_file=args.meta,
+        output_dir=args.output,
+    )
+
+    success = converter.convert()
+    exit(0 if success else 1)
 
 
 if __name__ == "__main__":
-    convert_to_huggingface()
+    main()

generation_config.json

@@ -1,9 +1,14 @@
 {
+  "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
   "pad_token_id": 4000,
+  "transformers_version": "4.56.0",
   "do_sample": true,
-  "max_length": 2048,
-  "temperature": 0.8,
-  "top_p": 0.9
+  "temperature": 0.7,
+  "top_p": 0.9,
+  "top_k": 40,
+  "max_new_tokens": 512,
+  "repetition_penalty": 1.1,
+  "no_repeat_ngram_size": 3
 }

pyproject.toml

@@ -11,11 +11,11 @@ authors = [
 keywords = ["tokenizer", "mon", "myanmar", "nlp", "huggingface", "sentencepiece"]
 
 dependencies = [
-    "transformers>=4.30.0",
-    "torch>=1.12.0",
-    "sentencepiece>=0.1.99",
-    "huggingface_hub>=0.15.0",
-    "protobuf>=3.20.0",
+    "transformers>=4.45.0",
+    "torch>=2.0.0",
+    "sentencepiece>=0.2.0",
+    "huggingface_hub>=0.24.0",
+    "protobuf>=4.21.0",
 ]
 
 [project.optional-dependencies]

sample_usage.py

@@ -1,22 +1,159 @@
+#!/usr/bin/env python3
+"""
+Sample usage examples for the Mon language tokenizer.
+
+This script demonstrates various ways to use the Mon tokenizer with
+Hugging Face Transformers library.
+"""
+
+import logging
+import time
+from typing import List, Dict, Any
+
+import torch
 from transformers import AutoTokenizer
 
-# Load the tokenizer
-tokenizer = AutoTokenizer.from_pretrained("janakhpon/mon_tokenizer")
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def basic_usage_example():
+    """Demonstrate basic tokenizer usage."""
+    print("=== Basic Usage Example ===")
+    
+    # Load the tokenizer
+    tokenizer = AutoTokenizer.from_pretrained("janakhpon/mon_tokenizer")
+    print(f"✓ Loaded tokenizer (vocab size: {tokenizer.vocab_size:,})")
+
+    # Example Mon texts
+    texts = [
+        "ဘာသာမန်",
+        "ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ်ရ။",
+        "ပ္ဍဲအခိင်မာံနဲသဵု မဒှ်ဘဝကွးဘာတက္ကသိုလ်ဂှ် ပါလုပ်ချဳဓရာင်ကၠုင် ပ္ဍဲပရေင်ကမၠောန်ယေန်သၞာင် ကေုာံ လိက်ပတ်မန် ဗွဲကတိုင်ကၟဟ်ရ။"
+    ]
+
+    for i, text in enumerate(texts, 1):
+        print(f"\nExample {i}:")
+        print(f"Input: {text}")
+        
+        # Tokenize the text
+        tokens = tokenizer(text, return_tensors="pt")
+        input_ids = tokens["input_ids"][0]
+
+        # Print results
+        print(f"Token IDs: {input_ids.tolist()}")
+        
+        # Convert to token strings
+        token_strings = tokenizer.convert_ids_to_tokens(input_ids)
+        print(f"Tokens: {token_strings}")
+
+        # Decode back to text
+        decoded = tokenizer.decode(input_ids, skip_special_tokens=True)
+        print(f"Decoded: {decoded}")
+        print(f"Round-trip success: {text == decoded}")
+
+
+def batch_processing_example():
+    """Demonstrate batch processing."""
+    print("\n=== Batch Processing Example ===")
+    
+    tokenizer = AutoTokenizer.from_pretrained("janakhpon/mon_tokenizer")
+    
+    # Multiple texts for batch processing
+    batch_texts = [
+        "ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ်ရ။",
+        "မန်တံဂှ် မံင်ပ္ဍဲ ရးမန် ကဵု ရးသေံ။",
+        "အရေဝ်ဘာသာမန် ပ္ဍဲလောကဏအ် ဂွံဆဵုကေတ်ရ။"
+    ]
+
+    # Batch tokenization with padding
+    batch_tokens = tokenizer(
+        batch_texts,
+        padding=True,
+        truncation=True,
+        return_tensors="pt",
+        max_length=128
+    )
+
+    print(f"Batch shape: {batch_tokens['input_ids'].shape}")
+    print(f"Attention mask shape: {batch_tokens['attention_mask'].shape}")
+    
+    # Process each item
+    for i, text in enumerate(batch_texts):
+        tokens_count = batch_tokens['attention_mask'][i].sum().item()
+        decoded = tokenizer.decode(batch_tokens['input_ids'][i], skip_special_tokens=True)
+        print(f"Text {i+1}: {tokens_count} tokens -> '{decoded}'")
+
+
+def advanced_features_example():
+    """Demonstrate advanced features."""
+    print("\n=== Advanced Features Example ===")
+    
+    tokenizer = AutoTokenizer.from_pretrained("janakhpon/mon_tokenizer")
+    text = "ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ်ရ။"
 
-# Example text
-text = "ပ္ဍဲအခိင်မာံနဲသဵု မဒှ်ဘဝကွးဘာတက္ကသိုလ်ဂှ် ပါလုပ်ချဳဓရာင်ကၠုင် ပ္ဍဲပရေင်ကမၠောန်ယေန်သၞာင် ကေုာံ လိက်ပတ်မန် ဗွဲကတိုင်ကၟဟ်ရ။"
+    # Different tokenization options
+    print("Special token handling:")
+    
+    # With special tokens
+    with_special = tokenizer(text, add_special_tokens=True, return_tensors="pt")
+    print(f"  With special tokens: {with_special['input_ids'].shape[1]} tokens")
+    
+    # Without special tokens  
+    without_special = tokenizer(text, add_special_tokens=False, return_tensors="pt")
+    print(f"  Without special tokens: {without_special['input_ids'].shape[1]} tokens")
+    
+    # Special token info
+    print(f"\nSpecial tokens:")
+    print(f"  BOS: '{tokenizer.bos_token}' (ID: {tokenizer.bos_token_id})")
+    print(f"  EOS: '{tokenizer.eos_token}' (ID: {tokenizer.eos_token_id})")
+    print(f"  UNK: '{tokenizer.unk_token}' (ID: {tokenizer.unk_token_id})")
+    print(f"  PAD: '{tokenizer.pad_token}' (ID: {tokenizer.pad_token_id})")
 
-# Tokenize the text
-tokens = tokenizer(text, return_tensors="pt")
-input_ids = tokens["input_ids"][0]
 
-# Print token IDs
-print("Token IDs:", input_ids.tolist())
+def performance_example():
+    """Demonstrate performance characteristics."""
+    print("\n=== Performance Example ===")
+    
+    tokenizer = AutoTokenizer.from_pretrained("janakhpon/mon_tokenizer")
+    
+    test_texts = [
+        ("Short", "ဘာသာမန်"),
+        ("Medium", "ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ်ရ။ မန်တံဂှ် မံင်ပ္ဍဲ ရးမန် ကဵု ရးသေံ။"),
+        ("Long", "အရေဝ်ဘာသာမန် ပ္ဍဲလောကဏအ် ဂွံဆဵုကေတ် ပ္ဍဲဍုင်သေံ ကဵု ဍုင်ဗၟာ ရ။ " * 10)
+    ]
+    
+    for name, text in test_texts:
+        char_count = len(text)
+        
+        # Measure tokenization time
+        start_time = time.time()
+        for _ in range(100):  # Average over 100 runs
+            tokens = tokenizer(text, return_tensors="pt")
+        avg_time = (time.time() - start_time) / 100
+        
+        token_count = tokens['input_ids'].shape[1] 
+        chars_per_sec = char_count / avg_time if avg_time > 0 else 0
+        
+        print(f"{name}: {char_count} chars -> {token_count} tokens")
+        print(f"  Time: {avg_time*1000:.2f}ms ({chars_per_sec:.0f} chars/sec)")
 
-# Print tokens
-token_list = tokenizer.convert_ids_to_tokens(input_ids)
-print("Tokens:", token_list)
 
-# Decode back to text
-decoded = tokenizer.decode(input_ids, skip_special_tokens=True)
-print("Decoded text:", decoded)
+if __name__ == "__main__":
+    print("🚀 Mon Tokenizer Usage Examples")
+    print("=" * 50)
+    
+    try:
+        basic_usage_example()
+        batch_processing_example() 
+        advanced_features_example()
+        performance_example()
+        
+        print(f"\n🎉 All examples completed successfully!")
+        print(f"\nFor more information, visit:")
+        print(f"https://huggingface.co/janakhpon/mon_tokenizer")
+        
+    except Exception as e:
+        print(f"❌ Error running examples: {e}")
+        exit(1)

test_tokenizer.py

@@ -1,107 +1,503 @@
 #!/usr/bin/env python3
-
 """
-test mon tokenizer hugging face integration
+Comprehensive test suite for Mon tokenizer Hugging Face integration.
+
+This script provides extensive testing for the Mon language tokenizer,
+including functionality tests, performance benchmarks, and compatibility checks.
 """
 
+import logging
+import time
+from pathlib import Path
+from typing import Dict, List, Tuple
+
 import torch
-from transformers import AutoTokenizer, GPT2LMHeadModel, GPT2Config
+from transformers import AutoTokenizer
 
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+logger = logging.getLogger(__name__)
 
-def test_tokenizer():
-    """test tokenizer loading and basic functionality"""
-    print("testing mon tokenizer")
-    
-    try:
-        tokenizer = AutoTokenizer.from_pretrained(".")
-        print(f"tokenizer loaded - vocab: {tokenizer.vocab_size:,}")
-        
-        # test tokenization
+
+class MonTokenizerTester:
+    """Comprehensive testing suite for Mon tokenizer."""
+
+    def __init__(self, tokenizer_path: str = "."):
+        """
+        Initialize the tester.
+
+        Args:
+            tokenizer_path: Path to the tokenizer files
+        """
+        self.tokenizer_path = tokenizer_path
+        self.tokenizer = None
+        self.test_results = {}
+
+    def load_tokenizer(self) -> bool:
+        """
+        Load the tokenizer for testing.
+
+        Returns:
+            bool: True if tokenizer loaded successfully, False otherwise
+        """
+        try:
+            logger.info(f"Loading tokenizer from: {self.tokenizer_path}")
+            self.tokenizer = AutoTokenizer.from_pretrained(
+                self.tokenizer_path,
+                local_files_only=True,
+                trust_remote_code=False
+            )
+            
+            logger.info(f"✓ Tokenizer loaded successfully")
+            logger.info(f"  - Vocabulary size: {self.tokenizer.vocab_size:,}")
+            logger.info(f"  - Model max length: {self.tokenizer.model_max_length:,}")
+            logger.info(f"  - Tokenizer class: {self.tokenizer.__class__.__name__}")
+            
+            return True
+
+        except Exception as e:
+            logger.error(f"✗ Failed to load tokenizer: {e}")
+            return False
+
+    def test_basic_functionality(self) -> bool:
+        """
+        Test basic tokenizer functionality.
+
+        Returns:
+            bool: True if all basic tests pass, False otherwise
+        """
+        logger.info("=== Testing Basic Functionality ===")
+
+        test_cases = [
+            {
+                "text": "ဘာသာမန်",
+                "description": "Single Mon word",
+                "expected_min_tokens": 1
+            },
+            {
+                "text": "ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ်ရ။",
+                "description": "Complete Mon sentence",
+                "expected_min_tokens": 3
+            },
+            {
+                "text": "မန်တံဂှ် မံင်ပ္ဍဲ ရးမန် ကဵု ရးသေံ။",
+                "description": "Mon geographical text",
+                "expected_min_tokens": 3
+            },
+            {
+                "text": "၁၂၃၄၅ ဂတာပ်ခ္ဍာ် ၂၀၂၄ သၞာံ",
+                "description": "Mon numerals and dates",
+                "expected_min_tokens": 2
+            },
+            {
+                "text": "အရေဝ်ဘာသာမန် ပ္ဍဲလောကဏအ် ဂွံဆဵုကေတ် ပ္ဍဲဍုင်သေံ ကဵု ဍုင်ဗၟာ ရ။",
+                "description": "Complex Mon linguistics text",
+                "expected_min_tokens": 5
+            }
+        ]
+
+        passed = 0
+        total = len(test_cases)
+
+        for i, test_case in enumerate(test_cases, 1):
+            text = test_case["text"]
+            description = test_case["description"]
+            expected_min_tokens = test_case["expected_min_tokens"]
+
+            try:
+                # Test encoding
+                start_time = time.time()
+                tokens = self.tokenizer(text, return_tensors="pt")
+                encoding_time = time.time() - start_time
+
+                # Test decoding
+                start_time = time.time()
+                decoded = self.tokenizer.decode(
+                    tokens["input_ids"][0], 
+                    skip_special_tokens=True
+                )
+                decoding_time = time.time() - start_time
+
+                # Validate results
+                token_count = tokens["input_ids"].shape[1]
+                round_trip_success = text.strip() == decoded.strip()
+
+                if token_count >= expected_min_tokens and round_trip_success:
+                    logger.info(f"✓ Test {i}: {description}")
+                    logger.info(f"  Tokens: {token_count}, Encoding: {encoding_time*1000:.2f}ms, "
+                               f"Decoding: {decoding_time*1000:.2f}ms")
+                    passed += 1
+                else:
+                    logger.warning(f"⚠ Test {i}: {description}")
+                    if token_count < expected_min_tokens:
+                        logger.warning(f"  Token count too low: {token_count} < {expected_min_tokens}")
+                    if not round_trip_success:
+                        logger.warning(f"  Round-trip failed:")
+                        logger.warning(f"    Input:  '{text}'")
+                        logger.warning(f"    Output: '{decoded}'")
+
+            except Exception as e:
+                logger.error(f"✗ Test {i}: {description} - ERROR: {e}")
+
+        success = passed == total
+        self.test_results["basic_functionality"] = {
+            "passed": passed,
+            "total": total,
+            "success": success
+        }
+
+        logger.info(f"Basic functionality: {passed}/{total} tests passed")
+        return success
+
+    def test_special_tokens(self) -> bool:
+        """
+        Test special token handling.
+
+        Returns:
+            bool: True if special token tests pass, False otherwise
+        """
+        logger.info("=== Testing Special Tokens ===")
+
+        try:
+            # Test special token IDs
+            special_tokens = {
+                "bos_token": self.tokenizer.bos_token,
+                "eos_token": self.tokenizer.eos_token,
+                "unk_token": self.tokenizer.unk_token,
+                "pad_token": self.tokenizer.pad_token,
+            }
+
+            special_token_ids = {
+                "bos_token_id": self.tokenizer.bos_token_id,
+                "eos_token_id": self.tokenizer.eos_token_id,
+                "unk_token_id": self.tokenizer.unk_token_id,
+                "pad_token_id": self.tokenizer.pad_token_id,
+            }
+
+            logger.info("Special tokens:")
+            for name, token in special_tokens.items():
+                token_id = special_token_ids[f"{name}_id"]
+                logger.info(f"  {name}: '{token}' (ID: {token_id})")
+
+            # Test that special tokens are properly handled
+            test_text = "ဘာသာမန်"
+            tokens_with_special = self.tokenizer(
+                test_text,
+                add_special_tokens=True,
+                return_tensors="pt"
+            )
+            tokens_without_special = self.tokenizer(
+                test_text,
+                add_special_tokens=False,
+                return_tensors="pt"
+            )
+
+            with_special_count = tokens_with_special["input_ids"].shape[1]
+            without_special_count = tokens_without_special["input_ids"].shape[1]
+
+            if with_special_count > without_special_count:
+                logger.info("✓ Special tokens are properly added")
+                success = True
+            else:
+                logger.warning("⚠ Special tokens may not be properly added")
+                success = False
+
+            self.test_results["special_tokens"] = {"success": success}
+            return success
+
+        except Exception as e:
+            logger.error(f"✗ Special token test failed: {e}")
+            self.test_results["special_tokens"] = {"success": False}
+            return False
+
+    def test_edge_cases(self) -> bool:
+        """
+        Test edge cases and error handling.
+
+        Returns:
+            bool: True if edge case tests pass, False otherwise
+        """
+        logger.info("=== Testing Edge Cases ===")
+
+        edge_cases = [
+            ("", "Empty string"),
+            ("   ", "Whitespace only"),
+            ("a", "Single ASCII character"),
+            ("123", "Numbers only"),
+            ("!@#$%", "Special characters only"),
+            ("ဘာသာမန်" * 100, "Very long text"),
+            ("ဟ", "Single Mon character"),
+            ("၀၁၂၃၄၅၆၇၈၉", "Mon numerals"),
+        ]
+
+        passed = 0
+        total = len(edge_cases)
+
+        for text, description in edge_cases:
+            try:
+                tokens = self.tokenizer(text, return_tensors="pt")
+                decoded = self.tokenizer.decode(tokens["input_ids"][0], skip_special_tokens=True)
+                
+                # For edge cases, we mainly check that no errors occur
+                logger.info(f"✓ {description}: {tokens['input_ids'].shape[1]} tokens")
+                passed += 1
+
+            except Exception as e:
+                logger.error(f"✗ {description}: {e}")
+
+        success = passed == total
+        self.test_results["edge_cases"] = {
+            "passed": passed,
+            "total": total,
+            "success": success
+        }
+
+        logger.info(f"Edge cases: {passed}/{total} tests passed")
+        return success
+
+    def test_performance_benchmark(self) -> bool:
+        """
+        Run performance benchmarks.
+
+        Returns:
+            bool: True if performance is acceptable, False otherwise
+        """
+        logger.info("=== Performance Benchmark ===")
+
+        # Test texts of varying lengths
         test_texts = [
             "ဘာသာမန်",
             "ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ်ရ။",
-            "မန်တံဂှ် မံင်ပ္ဍဲ ရးမန် ကဵု ရးသေံ။"
+            ("အရေဝ်ဘာသာမန် ပ္ဍဲလောကဏအ် ဂွံဆဵုကေတ် ပ္ဍဲဍုင်သေံ ကဵု ဍုင်ဗၟာ ရ။ " * 10),
+            ("မန်တံဂှ် မံင်ပ္ဍဲ ရးမန် ကဵု ရးသေံ။ " * 50),
         ]
-        
-        for text in test_texts:
-            inputs = tokenizer(text, return_tensors="pt")
-            decoded = tokenizer.decode(inputs["input_ids"][0], skip_special_tokens=True)
+
+        benchmark_results = []
+
+        for i, text in enumerate(test_texts, 1):
+            char_count = len(text)
             
-            print(f"input: '{text}'")
-            print(f"tokens: {inputs['input_ids'].shape}")
-            print(f"decoded: '{decoded}'")
-            print(f"round-trip: {'ok' if text == decoded else 'failed'}")
-            print()
-        
-        return True
-        
-    except Exception as e:
-        print(f"tokenizer test failed: {e}")
-        return False
+            # Benchmark encoding
+            start_time = time.time()
+            for _ in range(10):  # Run 10 times for average
+                tokens = self.tokenizer(text, return_tensors="pt")
+            encoding_time = (time.time() - start_time) / 10
 
+            # Benchmark decoding
+            start_time = time.time()
+            for _ in range(10):  # Run 10 times for average
+                decoded = self.tokenizer.decode(tokens["input_ids"][0])
+            decoding_time = (time.time() - start_time) / 10
 
-def test_model_integration():
-    """test tokenizer with gpt2 model"""
-    print("testing model integration")
-    
-    try:
-        tokenizer = AutoTokenizer.from_pretrained(".")
-        
-        # create small gpt2 model
-        config = GPT2Config(
-            vocab_size=tokenizer.vocab_size,
-            n_positions=512,
-            n_embd=256,
-            n_layer=4,
-            n_head=4,
-            bos_token_id=tokenizer.bos_token_id,
-            eos_token_id=tokenizer.eos_token_id,
-            pad_token_id=tokenizer.pad_token_id,
-        )
-        
-        model = GPT2LMHeadModel(config)
-        print(f"model created - params: {sum(p.numel() for p in model.parameters()):,}")
-        
-        # test generation
-        prompt = "ဘာသာမန်"
-        inputs = tokenizer(prompt, return_tensors="pt")
-        
-        with torch.no_grad():
-            outputs = model.generate(
-                **inputs,
-                max_length=inputs['input_ids'].shape[1] + 10,
-                do_sample=False,
-                pad_token_id=tokenizer.pad_token_id
+            token_count = tokens["input_ids"].shape[1]
+            
+            result = {
+                "text_length": char_count,
+                "token_count": token_count,
+                "encoding_time": encoding_time,
+                "decoding_time": decoding_time,
+                "chars_per_second": char_count / encoding_time if encoding_time > 0 else 0,
+                "tokens_per_second": token_count / decoding_time if decoding_time > 0 else 0
+            }
+            
+            benchmark_results.append(result)
+            
+            logger.info(f"Text {i} ({char_count} chars, {token_count} tokens):")
+            logger.info(f"  Encoding: {encoding_time*1000:.2f}ms ({result['chars_per_second']:.0f} chars/s)")
+            logger.info(f"  Decoding: {decoding_time*1000:.2f}ms ({result['tokens_per_second']:.0f} tokens/s)")
+
+        # Check if performance is acceptable (very lenient thresholds)
+        avg_encoding_time = sum(r["encoding_time"] for r in benchmark_results) / len(benchmark_results)
+        avg_decoding_time = sum(r["decoding_time"] for r in benchmark_results) / len(benchmark_results)
+
+        success = avg_encoding_time < 1.0 and avg_decoding_time < 1.0  # Less than 1 second average
+
+        self.test_results["performance"] = {
+            "avg_encoding_time": avg_encoding_time,
+            "avg_decoding_time": avg_decoding_time,
+            "success": success,
+            "details": benchmark_results
+        }
+
+        logger.info(f"Performance benchmark: {'PASSED' if success else 'FAILED'}")
+        return success
+
+    def test_compatibility(self) -> bool:
+        """
+        Test compatibility with transformers ecosystem.
+
+        Returns:
+            bool: True if compatibility tests pass, False otherwise
+        """
+        logger.info("=== Testing Compatibility ===")
+
+        try:
+            # Test tensor types
+            text = "ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ��ရ။"
+            
+            # Test different return types
+            tokens_pt = self.tokenizer(text, return_tensors="pt")
+            tokens_list = self.tokenizer(text, return_tensors=None)
+            
+            logger.info("✓ PyTorch tensor support")
+            logger.info("✓ List output support")
+
+            # Test padding and truncation
+            texts = [
+                "ဘာသာမန်",
+                "ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ်ရ။",
+                "မန်တံဂှ် မံင်ပ္ဍဲ ရးမန် ကဵု ရးသေံ။"
+            ]
+            
+            # Test batch processing
+            batch_tokens = self.tokenizer(
+                texts, 
+                padding=True, 
+                truncation=True, 
+                return_tensors="pt"
             )
+            
+            logger.info(f"✓ Batch processing: {batch_tokens['input_ids'].shape}")
+
+            # Test attention masks
+            if "attention_mask" in batch_tokens:
+                logger.info("✓ Attention mask generation")
+            else:
+                logger.warning("⚠ No attention mask generated")
+
+            success = True
+
+        except Exception as e:
+            logger.error(f"✗ Compatibility test failed: {e}")
+            success = False
+
+        self.test_results["compatibility"] = {"success": success}
+        return success
+
+    def run_all_tests(self) -> bool:
+        """
+        Run all test suites.
+
+        Returns:
+            bool: True if all tests pass, False otherwise
+        """
+        logger.info("🚀 Starting Mon Tokenizer Test Suite")
+        logger.info("=" * 50)
+
+        # Load tokenizer
+        if not self.load_tokenizer():
+            return False
+
+        # Run all test suites
+        test_suites = [
+            ("Basic Functionality", self.test_basic_functionality),
+            ("Special Tokens", self.test_special_tokens),
+            ("Edge Cases", self.test_edge_cases),
+            ("Performance Benchmark", self.test_performance_benchmark),
+            ("Compatibility", self.test_compatibility),
+        ]
+
+        results = []
+        for suite_name, test_func in test_suites:
+            logger.info(f"\n--- {suite_name} ---")
+            success = test_func()
+            results.append((suite_name, success))
+            logger.info(f"{suite_name}: {'✅ PASSED' if success else '❌ FAILED'}")
+
+        # Summary
+        logger.info("\n" + "=" * 50)
+        logger.info("📊 TEST SUMMARY")
+        logger.info("=" * 50)
+
+        passed_suites = sum(1 for _, success in results if success)
+        total_suites = len(results)
+
+        for suite_name, success in results:
+            status = "✅ PASSED" if success else "❌ FAILED"
+            logger.info(f"{suite_name}: {status}")
+
+        overall_success = passed_suites == total_suites
+        logger.info(f"\nOverall Result: {passed_suites}/{total_suites} test suites passed")
         
-        generated = tokenizer.decode(outputs[0], skip_special_tokens=True)
-        print(f"generated: '{generated}'")
-        
-        return True
+        if overall_success:
+            logger.info("🎉 ALL TESTS PASSED! Tokenizer is ready for production.")
+        else:
+            logger.error("⚠️  Some tests failed. Please review the issues above.")
+
+        return overall_success
+
+    def generate_test_report(self) -> str:
+        """
+        Generate a detailed test report.
+
+        Returns:
+            str: Formatted test report
+        """
+        if not self.test_results:
+            return "No test results available. Run tests first."
+
+        report = ["# Mon Tokenizer Test Report", ""]
         
-    except Exception as e:
-        print(f"model integration test failed: {e}")
-        return False
+        for test_name, result in self.test_results.items():
+            report.append(f"## {test_name.replace('_', ' ').title()}")
+            
+            if isinstance(result, dict) and "success" in result:
+                status = "✅ PASSED" if result["success"] else "❌ FAILED"
+                report.append(f"Status: {status}")
+                
+                if "passed" in result and "total" in result:
+                    report.append(f"Tests: {result['passed']}/{result['total']}")
+                
+            report.append("")
+
+        return "\n".join(report)
 
 
 def main():
-    """run all tests"""
-    print("mon tokenizer test suite")
-    
-    tests = [
-        ("tokenizer", test_tokenizer),
-        ("model integration", test_model_integration)
-    ]
-    
-    results = []
-    for name, test_func in tests:
-        print(f"\n--- {name} test ---")
-        success = test_func()
-        results.append(success)
-        print(f"{name}: {'passed' if success else 'failed'}")
-    
-    print(f"\ntest results: {sum(results)}/{len(results)} passed")
-    return all(results)
+    """Main entry point for the test script."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Test Mon tokenizer Hugging Face integration"
+    )
+    parser.add_argument(
+        "--tokenizer-path",
+        default=".",
+        help="Path to tokenizer files (default: current directory)",
+    )
+    parser.add_argument(
+        "--report",
+        action="store_true",
+        help="Generate detailed test report",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable verbose logging",
+    )
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    # Create tester and run tests
+    tester = MonTokenizerTester(tokenizer_path=args.tokenizer_path)
+    success = tester.run_all_tests()
+
+    # Generate report if requested
+    if args.report:
+        report = tester.generate_test_report()
+        report_path = Path("test_report.md")
+        with open(report_path, "w", encoding="utf-8") as f:
+            f.write(report)
+        logger.info(f"Test report saved to: {report_path}")
+
+    exit(0 if success else 1)
 
 
 if __name__ == "__main__":

tokenizer_config.json

@@ -1,19 +1,52 @@
 {
-  "model_type": "llama",
-  "tokenizer_class": "LlamaTokenizer",
-  "vocab_file": "mon_tokenizer.model",
-  "vocab_size": 4000,
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4000": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
   "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
-  "unk_token": "<unk>",
+  "legacy": true,
+  "model_max_length": 4096,
   "pad_token": "<pad>",
-  "bos_token_id": 1,
-  "eos_token_id": 2,
-  "unk_token_id": 0,
-  "pad_token_id": 4000,
-  "clean_up_tokenization_spaces": false,
   "sp_model_kwargs": {},
-  "add_bos_token": true,
-  "add_eos_token": false,
-  "model_max_length": 2048
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "vocab_file": "mon_tokenizer.model",
+  "vocab_size": 4000
 }

upload_to_hub.py

@@ -1,127 +1,423 @@
 #!/usr/bin/env python3
-
 """
-upload mon tokenizer to hugging face hub
+Upload Mon tokenizer to Hugging Face Hub.
+
+This script provides functionality to validate and upload the Mon language tokenizer
+to Hugging Face Hub with comprehensive validation and modern best practices.
+
+ required files:
+- `tokenizer_config.json` 
+- `special_tokens_map.json`
+- `generation_config.json` 
+- `README.md`
+- `.gitattributes`
+- `mon_tokenizer.model` (auto-detects either `tokenizer.model` or `mon_tokenizer.model`)
+
+Nothing missing - the script validates all files exist before upload and lists each file with size before uploading.
+
+The scripts now use modern conventions while maintaining backward compatibility through the `legacy: true` setting, which prevents the llama tokenizer warnings while using the latest transformers architecture.
+
 """
 
+import logging
 import os
 from pathlib import Path
-from huggingface_hub import HfApi, login
+from typing import List, Optional
+
+from huggingface_hub import HfApi, login, whoami
 from transformers import AutoTokenizer
 
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+logger = logging.getLogger(__name__)
 
-def validate_tokenizer(directory: str = ".") -> bool:
-    """validate tokenizer before upload"""
-    print("validating tokenizer")
-    
-    required_files = [
-        "mon_tokenizer.model",
-        "tokenizer_config.json", 
-        "special_tokens_map.json",
-        "README.md"
-    ]
-    
-    for file in required_files:
-        if not os.path.exists(os.path.join(directory, file)):
-            print(f"missing required file: {file}")
-            return False
-    
-    try:
-        tokenizer = AutoTokenizer.from_pretrained(directory)
-        test_text = "ဘာသာမန်"
-        tokens = tokenizer(test_text, return_tensors="pt")
-        decoded = tokenizer.decode(tokens["input_ids"][0], skip_special_tokens=True)
+
+class TokenizerUploader:
+    """Handles validation and upload of tokenizers to Hugging Face Hub."""
+
+    def __init__(self, directory: str = "."):
+        """
+        Initialize the uploader.
+
+        Args:
+            directory: Directory containing the tokenizer files
+        """
+        self.directory = Path(directory)
+        self.required_files = [
+            "tokenizer_config.json",
+            "special_tokens_map.json",
+            "generation_config.json",
+            "README.md",
+            ".gitattributes",
+        ]
+        # Check for either tokenizer.model or mon_tokenizer.model
+        self.model_files = ["tokenizer.model", "mon_tokenizer.model"]
+
+    def validate_files(self) -> bool:
+        """
+        Validate that all required files are present.
+
+        Returns:
+            bool: True if all files are present, False otherwise
+        """
+        logger.info(f"Validating tokenizer files in: {self.directory.absolute()}")
+
+        missing_files = []
+        present_files = []
+
+        # Check regular required files
+        for file_name in self.required_files:
+            file_path = self.directory / file_name
+            if file_path.exists():
+                size = file_path.stat().st_size
+                present_files.append((file_name, size))
+                logger.info(f"✓ {file_name} ({size:,} bytes)")
+            else:
+                missing_files.append(file_name)
+                logger.error(f"✗ {file_name} (missing)")
+
+        # Check for model file (either name is acceptable)
+        model_found = False
+        for model_name in self.model_files:
+            model_path = self.directory / model_name
+            if model_path.exists():
+                size = model_path.stat().st_size
+                present_files.append((model_name, size))
+                logger.info(f"✓ {model_name} ({size:,} bytes)")
+                model_found = True
+                break
         
-        if test_text != decoded:
-            print("tokenizer round-trip test failed")
+        if not model_found:
+            missing_files.append("tokenizer.model or mon_tokenizer.model")
+            logger.error(f"✗ Model file missing (looked for: {', '.join(self.model_files)})")
+
+        if missing_files:
+            logger.error(f"Missing required files: {', '.join(missing_files)}")
             return False
-        
-        print("validation passed")
+
+        logger.info(f"✓ All {len(self.required_files)} required files present")
         return True
+
+    def validate_tokenizer_functionality(self) -> bool:
+        """
+        Validate tokenizer functionality with comprehensive tests.
+
+        Returns:
+            bool: True if all tests pass, False otherwise
+        """
+        logger.info("Validating tokenizer functionality")
+
+        try:
+            # Load tokenizer with explicit local files only
+            abs_directory = str(self.directory.absolute())
+            tokenizer = AutoTokenizer.from_pretrained(
+                abs_directory, 
+                local_files_only=True,
+                trust_remote_code=False  # Security best practice
+            )
+
+            logger.info(f"✓ Tokenizer loaded (vocab: {tokenizer.vocab_size:,})")
+
+            # Comprehensive test cases for Mon language
+            test_cases = [
+                {
+                    "text": "ဘာသာမန်",
+                    "description": "Simple Mon word"
+                },
+                {
+                    "text": "ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ်ရ။",
+                    "description": "Complex Mon sentence with punctuation"
+                },
+                {
+                    "text": "မန်တံဂှ် မံင်ပ္ဍဲ ရးမန် ကဵု ရးသေံ။",
+                    "description": "Mon text with geographical references"
+                },
+                {
+                    "text": "၁၂၃၄၅ ဂတာပ်ခ္ဍာ် ၂၀၂၄ သၞာံ",
+                    "description": "Mon numerals and dates"
+                },
+                {
+                    "text": "",
+                    "description": "Empty string test"
+                }
+            ]
+
+            passed_tests = 0
+            
+            for i, test_case in enumerate(test_cases, 1):
+                text = test_case["text"]
+                description = test_case["description"]
+                
+                try:
+                    # Test tokenization
+                    tokens = tokenizer(text, return_tensors="pt")
+                    
+                    if tokens["input_ids"].numel() == 0 and text:
+                        logger.warning(f"⚠ Test {i}: Empty tokenization for non-empty text")
+                        continue
+                    
+                    # Test decoding
+                    decoded = tokenizer.decode(
+                        tokens["input_ids"][0], 
+                        skip_special_tokens=True
+                    )
+                    
+                    # Check round-trip accuracy
+                    if text.strip() == decoded.strip():
+                        logger.info(f"✓ Test {i}: {description} - PASSED")
+                        passed_tests += 1
+                    else:
+                        logger.warning(f"⚠ Test {i}: {description} - Round-trip mismatch")
+                        logger.warning(f"  Input:  '{text}'")
+                        logger.warning(f"  Output: '{decoded}'")
+                        
+                        # For some cases, minor differences might be acceptable
+                        if len(text.strip()) > 0:  # Don't fail on empty strings
+                            continue
+                        passed_tests += 1
+                        
+                except Exception as e:
+                    logger.error(f"✗ Test {i}: {description} - FAILED: {e}")
+                    return False
+
+            # Check test results
+            total_tests = len([tc for tc in test_cases if tc["text"]])  # Exclude empty string
+            if passed_tests >= total_tests - 1:  # Allow one test to fail
+                logger.info(f"✓ Functionality validation passed ({passed_tests}/{len(test_cases)} tests)")
+                return True
+            else:
+                logger.error(f"✗ Functionality validation failed ({passed_tests}/{len(test_cases)} tests passed)")
+                return False
+
+        except Exception as e:
+            logger.error(f"✗ Tokenizer validation failed: {e}")
+            return False
+
+    def validate_tokenizer(self) -> bool:
+        """
+        Run complete tokenizer validation.
+
+        Returns:
+            bool: True if validation passes, False otherwise
+        """
+        logger.info("=== Starting Tokenizer Validation ===")
         
-    except Exception as e:
-        print(f"validation failed: {e}")
-        return False
-
-
-def upload_to_hub(
-    repo_id: str = "janakhpon/mon_tokenizer",
-    directory: str = ".",
-    private: bool = False,
-    commit_message: str = "upload mon tokenizer"
-):
-    """upload tokenizer to hugging face hub"""
-    
-    print(f"uploading to {repo_id}")
-    
-    # validate first
-    if not validate_tokenizer(directory):
-        print("upload cancelled - validation failed")
-        return False
-    
-    try:
-        # login
-        print("logging in to hugging face")
-        login()
-        
-        # create api client
-        api = HfApi()
-        
-        # create/update repository
-        print(f"creating repository: {repo_id}")
-        api.create_repo(
-            repo_id=repo_id,
-            private=private,
-            exist_ok=True,
-            repo_type="model"
-        )
-        
-        # upload files
-        print("uploading files")
-        api.upload_folder(
-            folder_path=directory,
-            repo_id=repo_id,
-            commit_message=commit_message,
-            ignore_patterns=[
+        # Validate files
+        if not self.validate_files():
+            return False
+            
+        # Validate functionality
+        if not self.validate_tokenizer_functionality():
+            return False
+            
+        logger.info("✅ Tokenizer validation completed successfully")
+        return True
+
+    def check_authentication(self) -> Optional[str]:
+        """
+        Check Hugging Face authentication status.
+
+        Returns:
+            Optional[str]: Username if authenticated, None otherwise
+        """
+        try:
+            user_info = whoami()
+            username = user_info.get("name", "unknown")
+            logger.info(f"✓ Authenticated as: {username}")
+            return username
+        except Exception:
+            logger.warning("Not authenticated with Hugging Face")
+            return None
+
+    def upload_to_hub(
+        self,
+        repo_id: str,
+        private: bool = False,
+        commit_message: str = "Upload Mon language tokenizer",
+        create_pr: bool = False,
+    ) -> bool:
+        """
+        Upload tokenizer to Hugging Face Hub.
+
+        Args:
+            repo_id: Repository ID (e.g., "username/model-name")
+            private: Whether to create a private repository
+            commit_message: Commit message for the upload
+            create_pr: Whether to create a pull request instead of direct push
+
+        Returns:
+            bool: True if upload successful, False otherwise
+        """
+        logger.info(f"=== Starting Upload to {repo_id} ===")
+
+        try:
+            # Validate tokenizer first
+            if not self.validate_tokenizer():
+                logger.error("❌ Upload cancelled - validation failed")
+                return False
+
+            # Check authentication
+            if not self.check_authentication():
+                logger.info("Attempting to log in...")
+                try:
+                    login()
+                    if not self.check_authentication():
+                        logger.error("❌ Authentication failed")
+                        return False
+                except Exception as e:
+                    logger.error(f"❌ Login failed: {e}")
+                    return False
+
+            # Create API client
+            api = HfApi()
+
+            # Create/update repository
+            logger.info(f"Creating/updating repository: {repo_id}")
+            api.create_repo(
+                repo_id=repo_id,
+                private=private,
+                exist_ok=True,
+                repo_type="model"
+            )
+            logger.info("✓ Repository ready")
+
+            # List files to upload
+            upload_files = []
+            ignore_patterns = [
                 "*.pyc",
                 "__pycache__/",
                 ".git/",
                 ".venv/",
                 "*.lock",
-                "datasets/"
+                "uv.lock",
+                "pyproject.toml",
+                "datasets/",
+                "*.py",  # Don't upload Python scripts
+                "test_*",
+                "sample_*",
+                "convert_*",
+                "upload_*",
+                "*.meta.json"
             ]
-        )
-        
-        print(f"upload successful: https://huggingface.co/{repo_id}")
-        return True
-        
-    except Exception as e:
-        print(f"upload failed: {e}")
-        return False
+
+            logger.info("Files to be uploaded:")
+            for file_path in self.directory.iterdir():
+                if file_path.is_file() and not any(
+                    file_path.match(pattern) for pattern in ignore_patterns
+                ):
+                    size = file_path.stat().st_size
+                    upload_files.append(file_path.name)
+                    logger.info(f"  ✓ {file_path.name} ({size:,} bytes)")
+
+            # Upload files
+            logger.info("Uploading to Hugging Face Hub...")
+            api.upload_folder(
+                folder_path=str(self.directory),
+                repo_id=repo_id,
+                commit_message=commit_message,
+                ignore_patterns=ignore_patterns,
+                create_pr=create_pr,
+            )
+
+            hub_url = f"https://huggingface.co/{repo_id}"
+            logger.info(f"🎉 Upload successful!")
+            logger.info(f"📍 Repository URL: {hub_url}")
+            
+            if create_pr:
+                logger.info("📝 Pull request created for review")
+            
+            return True
+
+        except Exception as e:
+            logger.error(f"❌ Upload failed: {e}")
+            return False
 
 
 def main():
-    """main upload function"""
-    print("mon tokenizer hub uploader")
-    
-    # get repo info
-    repo_id = input("repository id (janakhpon/mon_tokenizer): ").strip()
-    if not repo_id:
-        repo_id = "janakhpon/mon_tokenizer"
-    
-    private = input("private repository? (y/n): ").strip().lower() == 'y'
-    
-    # upload
-    success = upload_to_hub(
-        repo_id=repo_id,
-        private=private,
-        commit_message="updated mon tokenizer"
+    """Main entry point for the upload script."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Upload Mon tokenizer to Hugging Face Hub"
+    )
+    parser.add_argument(
+        "--repo-id",
+        default="janakhpon/mon_tokenizer",
+        help="Repository ID (default: janakhpon/mon_tokenizer)",
+    )
+    parser.add_argument(
+        "--directory",
+        default=".",
+        help="Directory containing tokenizer files (default: current directory)",
+    )
+    parser.add_argument(
+        "--private",
+        action="store_true",
+        help="Create private repository",
+    )
+    parser.add_argument(
+        "--message",
+        default="Upload Mon language tokenizer",
+        help="Commit message",
+    )
+    parser.add_argument(
+        "--create-pr",
+        action="store_true",
+        help="Create pull request instead of direct push",
     )
-    
-    if success:
-        print("tokenizer successfully uploaded to hugging face hub")
+    parser.add_argument(
+        "--validate-only",
+        action="store_true",
+        help="Only validate tokenizer, don't upload",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable verbose logging",
+    )
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    # Create uploader
+    uploader = TokenizerUploader(directory=args.directory)
+
+    if args.validate_only:
+        # Only validate
+        success = uploader.validate_tokenizer()
+        logger.info("Validation completed.")
     else:
-        print("upload failed")
+        # Interactive mode if no repo ID provided
+        if args.repo_id == "janakhpon/mon_tokenizer":
+            print("\n🤗 Mon Tokenizer Hub Uploader")
+            print("=" * 40)
+            
+            repo_input = input(f"Repository ID [{args.repo_id}]: ").strip()
+            if repo_input:
+                args.repo_id = repo_input
+            
+            private_input = input("Private repository? (y/N): ").strip().lower()
+            args.private = private_input == 'y'
+            
+            print(f"\nUploading to: {args.repo_id}")
+            print(f"Private: {args.private}")
+            print("-" * 40)
+
+        # Upload tokenizer
+        success = uploader.upload_to_hub(
+            repo_id=args.repo_id,
+            private=args.private,
+            commit_message=args.message,
+            create_pr=args.create_pr,
+        )
+
+    exit(0 if success else 1)
 
 
 if __name__ == "__main__":

uv.lock

@@ -571,13 +571,13 @@ dev = [
 [package.metadata]
 requires-dist = [
     { name = "black", marker = "extra == 'dev'", specifier = ">=23.0.0" },
-    { name = "huggingface-hub", specifier = ">=0.15.0" },
+    { name = "huggingface-hub", specifier = ">=0.24.0" },
     { name = "isort", marker = "extra == 'dev'", specifier = ">=5.12.0" },
-    { name = "protobuf", specifier = ">=3.20.0" },
+    { name = "protobuf", specifier = ">=4.21.0" },
     { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" },
-    { name = "sentencepiece", specifier = ">=0.1.99" },
-    { name = "torch", specifier = ">=1.12.0" },
-    { name = "transformers", specifier = ">=4.30.0" },
+    { name = "sentencepiece", specifier = ">=0.2.0" },
+    { name = "torch", specifier = ">=2.0.0" },
+    { name = "transformers", specifier = ">=4.45.0" },
 ]
 provides-extras = ["dev"]