Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-09-02 03:59:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

pre commit

Browse Source
This commit is contained in:
xingjun.wang
2023-07-11 17:48:22 +08:00
parent 0c43a0e8ea
commit de32a8f3e6
1 changed files with 7 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
modelscope/msdatasets/download/dataset_builder.py
View File

@@ -223,18 +223,21 @@ class CsvDatasetBuilder(csv.Csv):
if field_name.endswith(':FILE'):
transform_fields.append(field_name)
base_extracted_dir: Union[str, list] = self.split_path_dict.get(split_name, '')
base_extracted_dir: Union[str, list] = self.split_path_dict.get(
split_name, '')
for field_name in transform_fields:
if isinstance(base_extracted_dir, list) and len(base_extracted_dir) > 0:
if isinstance(base_extracted_dir,
list) and len(base_extracted_dir) > 0:
if df.shape[0] != len(base_extracted_dir):
logger.error(
f"Number of lines in meta-csv file for split '{split_name}' ({df.shape[0]}) "
f"does not match number of data-files({len(base_extracted_dir)})!"
f'does not match number of data-files({len(base_extracted_dir)})!'
)
else:
df[field_name] = base_extracted_dir
elif isinstance(base_extracted_dir, str) and base_extracted_dir:
df[field_name] = df[field_name].apply(lambda x: os.path.join(base_extracted_dir, x))
df[field_name] = df[field_name].apply(
lambda x: os.path.join(base_extracted_dir, x))
else:
logger.warning(f'Nothing to do for field {field_name}')