Upload folder using huggingface_hub

README.md

@@ -69,6 +69,7 @@ config.text_config.head_dim = 32
 config.text_config.num_attention_heads = 1
 config.text_config.num_key_value_heads = 1
 config.text_config.num_hidden_layers = 2
+config.text_config.sliding_window_pattern = 2
 config.vision_config.hidden_size = 32
 config.vision_config.num_hidden_layers = 2
 config.vision_config.num_attention_heads = 1
@@ -76,6 +77,8 @@ config.vision_config.intermediate_size = 128
 model = Gemma3ForConditionalGeneration(
     config,
 ).to(torch.bfloat16)
+for layer in model.language_model.model.layers:
+    print(layer.is_sliding)
 model.generation_config = GenerationConfig.from_pretrained(
     source_model_id, trust_remote_code=True,
 )

config.json

@@ -37,7 +37,7 @@
     },
     "rope_theta": 1000000.0,
     "sliding_window": 1024,
-    "sliding_window_pattern": 6,
+    "sliding_window_pattern": 2,
     "use_cache": true,
     "vocab_size": 262208
   },