diff --git a/examples/pytorch/chatglm6b/finetune.py b/examples/pytorch/chatglm6b/finetune.py index 3fa73ba0..40eb8720 100644 --- a/examples/pytorch/chatglm6b/finetune.py +++ b/examples/pytorch/chatglm6b/finetune.py @@ -7,7 +7,7 @@ from chatglm_trainer import Seq2SeqTrainer from text_generation_metric import TextGenerationMetric from transformers import DataCollatorForSeq2Seq -from modelscope import snapshot_download +from modelscope import build_dataset_from_file, snapshot_download from modelscope.metainfo import Models from modelscope.models import Model from modelscope.msdatasets import MsDataset @@ -172,14 +172,20 @@ def cfg_modify_fn(cfg): return cfg -train_dataset = MsDataset.load( - args.train_dataset_name, - subset_name=args.train_subset_name, - split=args.train_split) -validation_dataset = MsDataset.load( - args.val_dataset_name, - subset_name=args.val_subset_name, - split=args.val_split) +if args.dataset_json_file is None: + train_dataset = MsDataset.load( + args.train_dataset_name, + subset_name=args.train_subset_name, + split=args.train_split, + namespace=args.train_dataset_namespace) + validation_dataset = MsDataset.load( + args.val_dataset_name, + subset_name=args.val_subset_name, + split=args.val_split, + namespace=args.val_dataset_namespace) +else: + train_dataset, validation_dataset = build_dataset_from_file( + args.dataset_json_file) model_dir = snapshot_download(args.model) model_config = read_config(model_dir) diff --git a/examples/pytorch/text_classification/finetune_text_classification.py b/examples/pytorch/text_classification/finetune_text_classification.py index e9c6eb27..111243f6 100644 --- a/examples/pytorch/text_classification/finetune_text_classification.py +++ b/examples/pytorch/text_classification/finetune_text_classification.py @@ -8,8 +8,14 @@ from modelscope.trainers import build_trainer def set_labels(labels): if isinstance(labels, str): - labels = labels.split(',') - return {label: id for id, label in enumerate(labels)} + label_list = labels.split(',') + else: + unique_labels = set(labels) + label_list = list(unique_labels) + label_list.sort() + label_list = list( + map(lambda x: x if isinstance(x, str) else str(x), label_list)) + return {label: id for id, label in enumerate(label_list)} @dataclass(init=False) @@ -52,7 +58,8 @@ class TextClassificationArguments(TrainingArgs): }) -config, args = TextClassificationArguments().parse_cli().to_config() +training_args = TextClassificationArguments().parse_cli() +config, args = training_args.to_config() print(config, args) @@ -62,6 +69,10 @@ def cfg_modify_fn(cfg): cfg.merge_from_dict(config) else: cfg = config + if training_args.labels is None: + labels = train_dataset[training_args.label] + validation_dataset[ + training_args.label] + cfg.merge_from_dict({'preprocessor.label2id': set_labels(labels)}) cfg.model['num_labels'] = len(cfg.preprocessor.label2id) if cfg.evaluation.period.eval_strategy == 'by_epoch': cfg.evaluation.period.by_epoch = True diff --git a/examples/pytorch/text_classification/run_train.sh b/examples/pytorch/text_classification/run_train.sh index e148655c..e05c71bc 100644 --- a/examples/pytorch/text_classification/run_train.sh +++ b/examples/pytorch/text_classification/run_train.sh @@ -9,8 +9,6 @@ PYTHONPATH=. python examples/pytorch/text_classification/finetune_text_classific --val_split 'validation' \ --first_sequence 'sentence' \ --label label \ - --num_labels 15 \ - --labels '0,1,2,3,4,5,6,7,8,9,10,11,12,13,14' \ --preprocessor 'sen-cls-tokenizer' \ --use_model_config True \ --max_epochs 1 \