missed ...
Browse files
src/axolotl/utils/data.py
CHANGED
|
@@ -62,7 +62,6 @@ def load_tokenized_prepared_datasets(
|
|
| 62 |
dataset = None
|
| 63 |
try:
|
| 64 |
if cfg.push_dataset_to_hub:
|
| 65 |
-
use_auth_token = True
|
| 66 |
dataset = load_dataset(
|
| 67 |
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=True
|
| 68 |
)
|
|
@@ -100,7 +99,7 @@ def load_tokenized_prepared_datasets(
|
|
| 100 |
d.path,
|
| 101 |
streaming=False,
|
| 102 |
data_files=d.data_files,
|
| 103 |
-
use_auth_token=
|
| 104 |
)
|
| 105 |
else:
|
| 106 |
ds = load_dataset(d.path, streaming=False, use_auth_token=True)
|
|
|
|
| 62 |
dataset = None
|
| 63 |
try:
|
| 64 |
if cfg.push_dataset_to_hub:
|
|
|
|
| 65 |
dataset = load_dataset(
|
| 66 |
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=True
|
| 67 |
)
|
|
|
|
| 99 |
d.path,
|
| 100 |
streaming=False,
|
| 101 |
data_files=d.data_files,
|
| 102 |
+
use_auth_token=True,
|
| 103 |
)
|
| 104 |
else:
|
| 105 |
ds = load_dataset(d.path, streaming=False, use_auth_token=True)
|