Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2025-12-25 04:29:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

add basic remap column wrapper

Browse Source 
Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/10539917

    * add basic remap column wrapper
This commit is contained in:
yingda.chen
2022-10-27 10:12:05 +08:00
parent 69104c0f8a
commit de708dd518
3 changed files with 32 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
modelscope/msdatasets/ms_dataset.py
View File

@@ -563,6 +563,18 @@ class MsDataset:
self._hf_ds.reset_format()
return self._hf_ds
def remap_columns(self, column_mapping: Dict[str, str]) -> Dataset:
"""
Rename columns and return the underlying hf dataset directly
TODO: support native MsDataset column rename.
Args:
column_mapping: the mapping of the original and new column names
Returns:
underlying hf dataset
"""
self._hf_ds.reset_format()
return self._hf_ds.rename_columns(column_mapping)
@staticmethod
def upload(object_name: str,
local_file_path: str,

21
tests/trainers/test_finetune_mplug.py
View File

@@ -24,17 +24,16 @@ class TestFinetuneMPlug(unittest.TestCase):
datadict = MsDataset.load(
'coco_captions_small_slice',
download_mode=DownloadMode.FORCE_REDOWNLOAD)
self.train_dataset = MsDataset(datadict['train'].to_hf_dataset().map(
lambda _: {
'question': 'what the picture describes?'
}).rename_column('image:FILE',
'image').rename_column('answer:Value', 'answer'))
self.test_dataset = MsDataset(datadict['test'].to_hf_dataset().map(
lambda _: {
'question': 'what the picture describes?'
}).rename_column('image:FILE',
'image').rename_column('answer:Value', 'answer'))
self.train_dataset = MsDataset(
datadict['train'].remap_columns({
'image:FILE': 'image',
'answer:Value': 'answer'
}).map(lambda _: {'question': 'what the picture describes?'}))
self.test_dataset = MsDataset(
datadict['test'].remap_columns({
'image:FILE': 'image',
'answer:Value': 'answer'
}).map(lambda _: {'question': 'what the picture describes?'}))
self.max_epochs = 2
def tearDown(self):

14
tests/trainers/test_finetune_text_generation.py
View File

@@ -130,10 +130,16 @@ class TestFinetuneTextGeneration(unittest.TestCase):
def test_finetune_cnndm(self):
from modelscope.msdatasets import MsDataset
dataset_dict = MsDataset.load('DuReader_robust-QG')
train_dataset = dataset_dict['train'].to_hf_dataset() \
.rename_columns({'text1': 'src_txt', 'text2': 'tgt_txt'})
eval_dataset = dataset_dict['validation'].to_hf_dataset() \
.rename_columns({'text1': 'src_txt', 'text2': 'tgt_txt'})
train_dataset = dataset_dict['train'].remap_columns({
'text1': 'src_txt',
'text2': 'tgt_txt'
})
eval_dataset = dataset_dict['validation'].remap_columns({
'text1':
'src_txt',
'text2':
'tgt_txt'
})
num_warmup_steps = 200
os.environ['LOCAL_RANK'] = '0'