From d38fa563a8111f5bf31d095aece8c9ee309a3418 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Fri, 19 May 2023 10:48:33 +0800 Subject: [PATCH] Re commit lossing files for flextrain Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12674746 --- .../finetune_text_generation.py | 22 ++++++++++++++----- .../pytorch/text_generation/run_train_gpt3.sh | 3 +++ .../pytorch/text_generation/run_train_mt5.sh | 3 +++ .../pytorch/text_generation/run_train_palm.sh | 3 +++ modelscope/trainers/trainer.py | 2 +- modelscope/trainers/training_args.py | 7 +++--- 6 files changed, 31 insertions(+), 9 deletions(-) diff --git a/examples/pytorch/text_generation/finetune_text_generation.py b/examples/pytorch/text_generation/finetune_text_generation.py index a89970e8..1631911e 100644 --- a/examples/pytorch/text_generation/finetune_text_generation.py +++ b/examples/pytorch/text_generation/finetune_text_generation.py @@ -1,6 +1,7 @@ from dataclasses import dataclass, field -from modelscope import EpochBasedTrainer, MsDataset, TrainingArgs +from modelscope import (EpochBasedTrainer, MsDataset, TrainingArgs, + build_dataset_from_file) from modelscope.metainfo import Trainers from modelscope.trainers import build_trainer @@ -94,14 +95,25 @@ def cfg_modify_fn(cfg): return cfg -dataset = MsDataset.load(args.train_dataset_name) -train_dataset = dataset['train'] -eval_dataset = dataset['validation' if 'validation' in dataset else 'test'] +if args.dataset_json_file is None: + train_dataset = MsDataset.load( + args.train_dataset_name, + subset_name=args.train_subset_name, + split=args.train_split, + namespace=args.train_dataset_namespace) + validation_dataset = MsDataset.load( + args.val_dataset_name, + subset_name=args.val_subset_name, + split=args.val_split, + namespace=args.val_dataset_namespace) +else: + train_dataset, validation_dataset = build_dataset_from_file( + args.dataset_json_file) kwargs = dict( model=args.model, train_dataset=train_dataset, - eval_dataset=eval_dataset, + eval_dataset=validation_dataset, seed=args.seed, work_dir=args.work_dir, cfg_modify_fn=cfg_modify_fn) diff --git a/examples/pytorch/text_generation/run_train_gpt3.sh b/examples/pytorch/text_generation/run_train_gpt3.sh index fd37b42c..78521951 100644 --- a/examples/pytorch/text_generation/run_train_gpt3.sh +++ b/examples/pytorch/text_generation/run_train_gpt3.sh @@ -9,6 +9,9 @@ PYTHONPATH=. torchrun --nproc_per_node $WORLD_SIZE examples/pytorch/text_generat --work_dir './tmp' \ --model 'damo/nlp_gpt3_text-generation_1.3B' \ --train_dataset_name 'chinese-poetry-collection' \ + --val_dataset_name 'chinese-poetry-collection' \ + --train_split 'train' \ + --val_split 'test' \ --preprocessor 'text-gen-jieba-tokenizer' \ --src_txt 'text1' \ --tgt_txt 'text2' \ diff --git a/examples/pytorch/text_generation/run_train_mt5.sh b/examples/pytorch/text_generation/run_train_mt5.sh index 6d032d6e..b2d0bbf1 100644 --- a/examples/pytorch/text_generation/run_train_mt5.sh +++ b/examples/pytorch/text_generation/run_train_mt5.sh @@ -4,6 +4,9 @@ PYTHONPATH=. torchrun examples/pytorch/text_generation/finetune_text_generation. --task 'text2text-generation' \ --model 'damo/nlp_mt5_zero-shot-augment_chinese-base' \ --train_dataset_name 'DuReader_robust-QG' \ + --val_dataset_name 'DuReader_robust-QG' \ + --train_split 'train' \ + --val_split 'validation' \ --src_txt 'text1' \ --tgt_txt 'text2' \ --max_epochs 1 \ diff --git a/examples/pytorch/text_generation/run_train_palm.sh b/examples/pytorch/text_generation/run_train_palm.sh index 68b9e89d..06153812 100644 --- a/examples/pytorch/text_generation/run_train_palm.sh +++ b/examples/pytorch/text_generation/run_train_palm.sh @@ -3,6 +3,9 @@ PYTHONPATH=. torchrun examples/pytorch/text_generation/finetune_text_generation. --work_dir './tmp' \ --model 'damo/nlp_palm2.0_pretrained_chinese-base' \ --train_dataset_name 'DuReader_robust-QG' \ + --val_dataset_name 'DuReader_robust-QG' \ + --train_split 'train' \ + --val_split 'validation' \ --src_txt 'text1' \ --tgt_txt 'text2' \ --max_epochs 1 \ diff --git a/modelscope/trainers/trainer.py b/modelscope/trainers/trainer.py index c980de04..278d6b44 100644 --- a/modelscope/trainers/trainer.py +++ b/modelscope/trainers/trainer.py @@ -155,10 +155,10 @@ class EpochBasedTrainer(BaseTrainer): self.cfg_modify_fn = cfg_modify_fn # add default config merge_cfg(self.cfg) - self.cfg = self.rebuild_config(self.cfg) if 'cfg_options' in kwargs: self.cfg.merge_from_dict(kwargs['cfg_options']) self.cfg = update_cfg(self.cfg) + self.cfg = self.rebuild_config(self.cfg) if isinstance(model, (TorchModel, nn.Module)): self.model = model diff --git a/modelscope/trainers/training_args.py b/modelscope/trainers/training_args.py index b7236163..16453d7c 100644 --- a/modelscope/trainers/training_args.py +++ b/modelscope/trainers/training_args.py @@ -9,6 +9,7 @@ import json from modelscope.trainers.cli_argument_parser import CliArgumentParser from modelscope.utils.config import Config +from modelscope.utils.constant import DEFAULT_DATASET_NAMESPACE def set_flatten_value(values: Union[str, List[str]]): @@ -62,13 +63,13 @@ class DatasetArgs: }) train_dataset_namespace: str = field( - default=None, + default=DEFAULT_DATASET_NAMESPACE, metadata={ 'help': 'The dataset namespace used for training', }) val_dataset_namespace: str = field( - default=None, + default=DEFAULT_DATASET_NAMESPACE, metadata={ 'help': 'The dataset namespace used for evaluating', }) @@ -450,7 +451,7 @@ class TrainingArgs(DatasetArgs, TrainArgs, ModelArgs): _unknown[unknown[i].replace('-', '')] = parse_value(unknown[i + 1]) args_dict = vars(args) self.manual_args += parser.manual_args - + self._unknown_args.update(_unknown) for key, value in deepcopy(args_dict).items(): if key is not None and hasattr(self, key): setattr(self, key, value)