tuanpasg/mb_llama_iso_0.4

Merged Model

• Base model: meta-llama/Llama-3.2-3B
• Algorithm: IsoCTS
• Save path: ./merged_models/Llama-3.2-3B_merged/IsoCTS_scaling_coef_1_common_space_fraction_0.8_validate_gen_flag_True_validate_scaling_start_0.0_validate_scaling_stop_1.0_validate_scaling_step_0.2
• Fine-tuned checkpoints: ['MergeBench/Llama-3.2-3B_instruction', 'MergeBench/Llama-3.2-3B_math', 'MergeBench/Llama-3.2-3B_coding']
• Merge runtime (s): 4549.626

Arguments

{
  "local_rank": 0,
  "scaling_coef": 1,
  "K": 0.2,
  "merge_func": "sum",
  "p": 0.8,
  "sigmoid_bias": 3,
  "sparsity": 0.1,
  "lr": 100000000.0,
  "n_epochs": 1,
  "l1_strength": 0.0,
  "dataless": false,
  "k": 2,
  "lamda": [
    0.5
  ],
  "lamda_tuning": false,
  "task_names": null,
  "reduction": null,
  "fisher_only": false,
  "merge_only": false,
  "save_group": null,
  "model_coeff_value": 0.3,
  "keep_checkpoints": false,
  "beta_coef": 1,
  "n": 64,
  "m": 256,
  "pruned_subdir": "pruned_models",
  "save_pruned_models": true,
  "warn_min_overlap_fallback": true,
  "common_space_fraction": 0.8,
  "validate_gen_flag": true,
  "validate_scaling_start": 0.0,
  "validate_scaling_stop": 1.0,
  "validate_scaling_step": 0.2,
  "base_model": "meta-llama/Llama-3.2-3B",
  "algo": "IsoCTS",
  "save_path": "./merged_models/"
}

Merge kwargs

{
  "scaling_coef": 0.4,
  "common_space_fraction": 0.8,
  "validate_gen_flag": true,
  "validate_scaling_start": 0.0,
  "validate_scaling_stop": 1.0,
  "validate_scaling_step": 0.2
}