Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2025-12-25 12:39:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

[to #42322933] fix for plug

Browse Source 
1. update pipeline for new preprocessor
2. update trainer for dist_info (remove megatron-ddp wapper)

Link: https://code.aone.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/10969594
This commit is contained in:
suluyan.sly
2022-12-05 14:34:21 +08:00
parent 9d43823f36
commit c36b1cce8a
2 changed files with 6 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
modelscope/pipelines/nlp/distributed_plug_pipeline.py
View File

@@ -65,8 +65,7 @@ class DistributedPlugPipeline(DistributedPipeline):
sequence_length=sequence_length,
**kwargs)
super().__init__(model, preprocessor=preprocessor, **kwargs)
assert hasattr(preprocessor, 'tokenizer')
self.cls_token_id = preprocessor.tokenizer.cls_token_id
self.cls_token_id = preprocessor.nlp_tokenizer.tokenizer.cls_token_id
@classmethod
def _forward_one(cls, inputs: Dict[str, Any]) -> Dict[str, Any]:
@@ -105,6 +104,6 @@ class DistributedPlugPipeline(DistributedPipeline):
from modelscope.outputs import OutputKeys
generate_context = inputs['generate_context']
generate_context = ''.join(
self.preprocessor.tokenizer.convert_ids_to_tokens(
self.preprocessor.nlp_tokenizer.tokenizer.convert_ids_to_tokens(
generate_context)).replace('[UNK]', '“').replace('##', '')
return {OutputKeys.TEXT: generate_context}

8
modelscope/trainers/nlp/plug_trainer.py
View File

@@ -66,9 +66,9 @@ class PlugTrainer(NlpEpochBasedTrainer):
from deepspeed.ops.adam import DeepSpeedCPUAdam
model = self.model
embeddings = model.module.module.model.bert.embeddings
layers = model.module.module.model.bert.encoder.layer
dec_layers = model.module.module.model.decoder.decoder
embeddings = model.module.model.bert.embeddings
layers = model.module.model.bert.encoder.layer
dec_layers = model.module.model.decoder.decoder
param_groups = []
param_groups += list(
self._get_params_for_weight_decay_optimization(layers))
@@ -160,7 +160,7 @@ class PlugTrainer(NlpEpochBasedTrainer):
def evaluation_step(self, data):
# wapper 1: DeepspeedEngine, wapper 2: DDP
model = self.model.module.module
model = self.model.module
model.eval()
# model: fp16 wapper; model.module : distributedPlug