Upload config.json

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/home/users/astar/ares/stunvat/scratch/checkpoints/moe-llava-qwen1.5-1.8b-ft-mergedlora",
+  "_name_or_path": "Qwen/Qwen1.5-1.8B",
   "architectures": [
     "LlavaQwen1_5ForCausalLM"
   ],
@@ -13,63 +13,17 @@
   "image_projector_type": "mlp2x_gelu",
   "initializer_range": 0.02,
   "intermediate_size": 5504,
-  "lora": {},
   "max_position_embeddings": 32768,
   "max_window_layers": 21,
   "mm_hidden_size": 768,
   "mm_image_tower": "google/siglip-base-patch16-256-multilingual",
-  "mm_projector_lr": null,
+  "mm_projector_lr": 1.25e-06,
   "mm_use_im_patch_token": false,
   "mm_use_im_start_end": false,
   "mm_video_tower": null,
   "mm_vision_select_feature": "patch",
   "mm_vision_select_layer": -2,
-  "model_type": "moe_llava_qwen1_5",
-  "moe": {
-    "capacity_factor": 1.5,
-    "ep_size": 1,
-    "eval_capacity_factor": 2.0,
-    "min_capacity": 0,
-    "moe_enable": true,
-    "moe_layers_idx": [
-      0,
-      2,
-      4,
-      6,
-      8,
-      10,
-      12,
-      14,
-      16,
-      18,
-      20,
-      22
-    ],
-    "moe_mode": "sparse",
-    "num_experts": [
-      4,
-      4,
-      4,
-      4,
-      4,
-      4,
-      4,
-      4,
-      4,
-      4,
-      4,
-      4
-    ],
-    "router_aux_loss_coef": 0.01,
-    "top_k_experts": 2,
-    "train_modules": [
-      "mlp.gate_proj",
-      "mlp.up_proj",
-      "mlp.down_proj",
-      "wg"
-    ],
-    "use_residual": false
-  },
+  "model_type": "llava_qwen1_5",
   "num_attention_heads": 16,
   "num_hidden_layers": 24,
   "num_key_value_heads": 16,
@@ -80,7 +34,7 @@
   "tie_word_embeddings": false,
   "tokenizer_padding_side": "right",
   "torch_dtype": "float16",
-  "transformers_version": "4.37.0",
+  "transformers_version": "4.42.4",
   "tune_mm_mlp_adapter": false,
   "use_cache": true,
   "use_mm_proj": true,