mirror of
https://github.com/modelscope/modelscope.git
synced 2026-09-01 19:49:03 +02:00
[to #42322933] Refine the nlp examples of finetuning
1. Refine the nlp finetune examples
2. Remove some useless code
Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9631158
This commit is contained in:
@@ -517,10 +517,3 @@ class MsDataset:
|
||||
def to_hf_dataset(self) -> Dataset:
|
||||
self._hf_ds.reset_format()
|
||||
return self._hf_ds
|
||||
|
||||
@staticmethod
|
||||
def interleave_datasets(datasets: List[Any],
|
||||
probabilities: Optional[List[float]] = None,
|
||||
seed: Optional[int] = None):
|
||||
from datasets import interleave_datasets
|
||||
return interleave_datasets(datasets, probabilities, seed)
|
||||
|
||||
@@ -32,7 +32,6 @@ class EvaluationHook(Hook):
|
||||
def do_evaluate(self, trainer):
|
||||
"""Evaluate the results."""
|
||||
eval_res = trainer.evaluate()
|
||||
trainer.data_loader = trainer.train_dataloader
|
||||
for name, val in eval_res.items():
|
||||
trainer.log_buffer.output[name] = val
|
||||
|
||||
|
||||
@@ -23,7 +23,6 @@ class TestTrainerWithNlp(unittest.TestCase):
|
||||
if not os.path.exists(self.tmp_dir):
|
||||
os.makedirs(self.tmp_dir)
|
||||
|
||||
# todo: Replace below scripts with MsDataset.load when the formal dataset service is ready
|
||||
self.dataset = MsDataset.load(
|
||||
'afqmc_small', namespace='userxiaoming', split='train')
|
||||
|
||||
|
||||
Reference in New Issue
Block a user