Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2025-12-25 04:29:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

add five finetune task & merge master

Browse Source
This commit is contained in:
翎航
2022-11-08 10:39:45 +08:00
parent 04377cfd79
commit a02b2409d8
2 changed files with 10 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
modelscope/preprocessors/ofa/summarization.py
View File

@@ -46,7 +46,6 @@ class OfaSummarizationPreprocessor(OfaBasePreprocessor):
def _build_infer_sample(self, data: Dict[str, Any]) -> Dict[str, Any]:
source = super().pre_caption(
data[self.column_map['text']], max_words=self.max_src_length)
# source = source.strip()[:self.max_src_length]
source = source.replace('[unk]', 'unk').replace('<unk>', 'unk')
prompt = self.cfg.model.get(
'prompt', ' " {} " Summarize the article with a title: ')

13
tests/trainers/test_ofa_trainer.py
View File

@@ -71,13 +71,20 @@ class TestOfaTrainer(unittest.TestCase):
@unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
def test_trainer_std(self):
# WORKSPACE = './workspace/ckpts/recognition'
# os.makedirs(WORKSPACE, exist_ok=True)
#
# pretrained_model = 'damo/ofa_ocr-recognition_scene_base_zh'
# cfg = read_config(pretrained_model)
# config_file = os.path.join(WORKSPACE, ModelFile.CONFIGURATION)
# cfg.dump(config_file)
WORKSPACE = './workspace/ckpts/recognition'
os.makedirs(WORKSPACE, exist_ok=True)
config_file = os.path.join(WORKSPACE, ModelFile.CONFIGURATION)
with open(config_file, 'w') as writer:
json.dump(self.finetune_cfg, writer)
pretrained_model = 'damo/ofa_ocr-recognition_scene_base_zh'
cfg = read_config(pretrained_model)
config_file = os.path.join(WORKSPACE, ModelFile.CONFIGURATION)
cfg.dump(config_file)
args = dict(
model=pretrained_model,