BounharAbdelaziz/Transliteration-Moroccan-Darija

README.md

@@ -30,14 +30,16 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 0.05
-- train_batch_size: 512
-- eval_batch_size: 512
+- learning_rate: 3e-05
+- train_batch_size: 128
+- eval_batch_size: 128
 - seed: 42
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 256
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
-- lr_scheduler_warmup_ratio: 0.01
-- num_epochs: 30
+- lr_scheduler_warmup_ratio: 0.02
+- num_epochs: 120
 
 ### Framework versions
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "./checkpoint-1500/",
+  "_name_or_path": "./checkpoint-5000/",
   "architectures": [
     "EncoderDecoderModel"
   ],
@@ -26,7 +26,7 @@
     "forced_eos_token_id": null,
     "hidden_act": "gelu",
     "hidden_dropout_prob": 0.1,
-    "hidden_size": 64,
+    "hidden_size": 256,
     "id2label": {
       "0": "LABEL_0",
       "1": "LABEL_1"
@@ -46,10 +46,10 @@
     "min_length": 0,
     "model_type": "bert",
     "no_repeat_ngram_size": 0,
-    "num_attention_heads": 2,
+    "num_attention_heads": 8,
     "num_beam_groups": 1,
     "num_beams": 1,
-    "num_hidden_layers": 2,
+    "num_hidden_layers": 4,
     "num_return_sequences": 1,
     "output_attentions": false,
     "output_hidden_states": false,
@@ -105,7 +105,7 @@
     "forced_eos_token_id": null,
     "hidden_act": "gelu",
     "hidden_dropout_prob": 0.1,
-    "hidden_size": 64,
+    "hidden_size": 256,
     "id2label": {
       "0": "LABEL_0",
       "1": "LABEL_1"
@@ -125,10 +125,10 @@
     "min_length": 0,
     "model_type": "bert",
     "no_repeat_ngram_size": 0,
-    "num_attention_heads": 2,
+    "num_attention_heads": 8,
     "num_beam_groups": 1,
     "num_beams": 1,
-    "num_hidden_layers": 2,
+    "num_hidden_layers": 4,
     "num_return_sequences": 1,
     "output_attentions": false,
     "output_hidden_states": false,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:816a30480e3c6c9dbbbb8beda170def7573bba47ecbd7ea18790f9829eadfcae
-size 22805496
+oid sha256:06752f7d26b1fd95377b38f6cc6c2be1ca93b2d04222f65053155b794e6ff532
+size 127350216

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:000459e2b1cc8e87d2d315734c195c15b6a59ac770c9921ddf85433e87be1519
+oid sha256:b6927d6e7c46d3c0f554f1706565c3cbaedeb09f7598e1c70acfb2f6186fba18
 size 5048