[to #42322933] Refine the nlp examples of finetuning

1. Refine the nlp finetune examples
2. Remove some useless code
        Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9631158
This commit is contained in:
yuze.zyz
2022-08-03 20:31:10 +08:00
parent 06a761ab70
commit 064f1041a9
3 changed files with 0 additions and 9 deletions

View File

@@ -517,10 +517,3 @@ class MsDataset:
def to_hf_dataset(self) -> Dataset:
self._hf_ds.reset_format()
return self._hf_ds
@staticmethod
def interleave_datasets(datasets: List[Any],
probabilities: Optional[List[float]] = None,
seed: Optional[int] = None):
from datasets import interleave_datasets
return interleave_datasets(datasets, probabilities, seed)

View File

@@ -32,7 +32,6 @@ class EvaluationHook(Hook):
def do_evaluate(self, trainer):
"""Evaluate the results."""
eval_res = trainer.evaluate()
trainer.data_loader = trainer.train_dataloader
for name, val in eval_res.items():
trainer.log_buffer.output[name] = val

View File

@@ -23,7 +23,6 @@ class TestTrainerWithNlp(unittest.TestCase):
if not os.path.exists(self.tmp_dir):
os.makedirs(self.tmp_dir)
# todo: Replace below scripts with MsDataset.load when the formal dataset service is ready
self.dataset = MsDataset.load(
'afqmc_small', namespace='userxiaoming', split='train')