This commit is contained in:
suluyana
2024-10-17 11:16:40 +08:00
parent 98832eaea0
commit 9300f8de65
2 changed files with 4 additions and 2 deletions

View File

@@ -17,7 +17,8 @@ class DatasetContextConfig:
data_files: Union[str, Sequence[str],
Mapping[str, Union[str, Sequence[str]]]],
download_mode: DownloadMode, cache_root_dir: str,
use_streaming: bool, stream_batch_size: int, trust_remote_code: bool, **kwargs):
use_streaming: bool, stream_batch_size: int,
trust_remote_code: bool, **kwargs):
self._download_config = None
self._data_meta_config = None

View File

@@ -121,7 +121,8 @@ class GeneralMsDatasetTest(unittest.TestCase):
f'please check the url: {py_script_url}'
# Load the dataset
ds = MsDataset.load(py_script_file, subset_name='cola', split='train')
ds = MsDataset.load(
py_script_file, subset_name='cola', split='train')
sample = next(iter(ds))
logger.info(f'>>output of test_local_py_script:\n {sample}')
assert sample