always drop samples that are too long (#452)

src/axolotl/utils/trainer.py

@@ -284,15 +284,15 @@ def disable_datasets_caching():
 
 
 def process_datasets_for_packing(cfg, train_dataset, eval_dataset):
+    drop_long = partial(drop_long_seq, sequence_len=cfg.sequence_len)
+    train_dataset = train_dataset.filter(drop_long, num_proc=os.cpu_count())
+    if eval_dataset:
+        eval_dataset = eval_dataset.filter(drop_long, num_proc=os.cpu_count())
+
     if cfg.sample_packing:
-        drop_long = partial(drop_long_seq, sequence_len=cfg.sequence_len)
-        train_dataset = train_dataset.filter(drop_long, num_proc=os.cpu_count()).map(
-            add_position_ids, num_proc=os.cpu_count()
-        )
+        train_dataset = train_dataset.map(add_position_ids, num_proc=os.cpu_count())
         if eval_dataset:
-            eval_dataset = eval_dataset.filter(drop_long, num_proc=os.cpu_count()).map(
-                add_position_ids, num_proc=os.cpu_count()
-            )
+            eval_dataset = eval_dataset.map(add_position_ids, num_proc=os.cpu_count())
     return train_dataset, eval_dataset