Fix: Future deprecation warning with use_auth_token (#680)
Browse files
src/axolotl/utils/data.py
CHANGED
|
@@ -114,7 +114,7 @@ def load_tokenized_prepared_datasets(
|
|
| 114 |
if cfg.push_dataset_to_hub:
|
| 115 |
dataset = load_dataset(
|
| 116 |
f"{cfg.push_dataset_to_hub}/{ds_hash}",
|
| 117 |
-
|
| 118 |
)
|
| 119 |
dataset = dataset["train"]
|
| 120 |
except Exception: # pylint: disable=broad-except # nosec
|
|
@@ -155,7 +155,7 @@ def load_tokenized_prepared_datasets(
|
|
| 155 |
d.path,
|
| 156 |
name=d.name,
|
| 157 |
streaming=True,
|
| 158 |
-
|
| 159 |
)
|
| 160 |
ds_from_hub = True
|
| 161 |
except FileNotFoundError:
|
|
@@ -202,7 +202,7 @@ def load_tokenized_prepared_datasets(
|
|
| 202 |
name=d.name,
|
| 203 |
streaming=False,
|
| 204 |
data_files=d.data_files,
|
| 205 |
-
|
| 206 |
)
|
| 207 |
else:
|
| 208 |
if isinstance(d.data_files, str):
|
|
@@ -417,7 +417,7 @@ def load_prepare_datasets(
|
|
| 417 |
)
|
| 418 |
dataset = load_dataset(
|
| 419 |
f"{cfg.push_dataset_to_hub}/{ds_hash}",
|
| 420 |
-
|
| 421 |
)
|
| 422 |
dataset = dataset["train"]
|
| 423 |
except Exception: # pylint: disable=broad-except # nosec
|
|
|
|
| 114 |
if cfg.push_dataset_to_hub:
|
| 115 |
dataset = load_dataset(
|
| 116 |
f"{cfg.push_dataset_to_hub}/{ds_hash}",
|
| 117 |
+
token=use_auth_token,
|
| 118 |
)
|
| 119 |
dataset = dataset["train"]
|
| 120 |
except Exception: # pylint: disable=broad-except # nosec
|
|
|
|
| 155 |
d.path,
|
| 156 |
name=d.name,
|
| 157 |
streaming=True,
|
| 158 |
+
token=use_auth_token,
|
| 159 |
)
|
| 160 |
ds_from_hub = True
|
| 161 |
except FileNotFoundError:
|
|
|
|
| 202 |
name=d.name,
|
| 203 |
streaming=False,
|
| 204 |
data_files=d.data_files,
|
| 205 |
+
token=use_auth_token,
|
| 206 |
)
|
| 207 |
else:
|
| 208 |
if isinstance(d.data_files, str):
|
|
|
|
| 417 |
)
|
| 418 |
dataset = load_dataset(
|
| 419 |
f"{cfg.push_dataset_to_hub}/{ds_hash}",
|
| 420 |
+
token=use_auth_token,
|
| 421 |
)
|
| 422 |
dataset = dataset["train"]
|
| 423 |
except Exception: # pylint: disable=broad-except # nosec
|