mirror of
https://github.com/modelscope/modelscope.git
synced 2025-12-16 16:27:45 +01:00
1dbff6cb48
Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/13071970 * support jsonl in meta * add UT and refine fetch_meta_files_from_url
31 lines
827 B
Python
31 lines
827 B
Python
# Copyright (c) Alibaba, Inc. and its affiliates.
|
|
|
|
import unittest
|
|
|
|
from modelscope.msdatasets import MsDataset
|
|
from modelscope.utils import logger as logging
|
|
from modelscope.utils.constant import DownloadMode
|
|
from modelscope.utils.test_utils import test_level
|
|
|
|
logger = logging.get_logger()
|
|
|
|
|
|
class TestLoadMetaJsonl(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.dataset_id = 'modelscope/ms_ds_meta_jsonlines'
|
|
|
|
@unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
|
|
def test_load_jsonl_in_meta(self):
|
|
ds = MsDataset.load(
|
|
self.dataset_id,
|
|
split='test',
|
|
download_mode=DownloadMode.FORCE_REDOWNLOAD)
|
|
ds_one = next(iter(ds))
|
|
logger.info(next(iter(ds)))
|
|
assert ds_one['text']
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|