Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-09-01 19:49:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

delete file about gen

Browse Source
This commit is contained in:
ly119399
2022-06-13 16:46:59 +08:00
parent ccbbd959a7
commit a09fec6c56
7 changed files with 0 additions and 373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
modelscope/models/nlp/space/dialog_generation_model.py
View File

@@ -1,103 +0,0 @@
from typing import Any, Dict, Optional
from modelscope.trainers.nlp.space.trainers.gen_trainer import MultiWOZTrainer
from modelscope.utils.constant import Tasks
from ...base import Model, Tensor
from ...builder import MODELS
from .model.generator import Generator
from .model.model_base import ModelBase
__all__ = ['DialogGenerationModel']
@MODELS.register_module(
Tasks.dialog_generation, module_name=r'space-generation')
class DialogGenerationModel(Model):
def __init__(self, model_dir: str, *args, **kwargs):
"""initialize the test generation model from the `model_dir` path.
Args:
model_dir (str): the model path.
model_cls (Optional[Any], optional): model loader, if None, use the
default loader to load model weights, by default None.
"""
super().__init__(model_dir, *args, **kwargs)
self.model_dir = model_dir
self.text_field = kwargs.pop('text_field')
self.config = kwargs.pop('config')
self.generator = Generator.create(self.config, reader=self.text_field)
self.model = ModelBase.create(
model_dir=model_dir,
config=self.config,
reader=self.text_field,
generator=self.generator)
def to_tensor(array):
"""
numpy array -> tensor
"""
import torch
array = torch.tensor(array)
return array.cuda() if self.config.use_gpu else array
self.trainer = MultiWOZTrainer(
model=self.model,
to_tensor=to_tensor,
config=self.config,
reader=self.text_field,
evaluator=None)
self.trainer.load()
def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]:
"""return the result by the model
Args:
input (Dict[str, Any]): the preprocessed data
Returns:
Dict[str, np.ndarray]: results
Example:
{
'predictions': array([1]), # lable 0-negative 1-positive
'probabilities': array([[0.11491239, 0.8850876 ]], dtype=float32),
'logits': array([[-0.53860897, 1.5029076 ]], dtype=float32) # true value
}
"""
from numpy import array, float32
import torch
# turn_1 = {
# 'user': [
# 13, 1045, 2052, 2066, 1037, 10095, 2013, 3002, 2198, 1005,
# 1055, 2267, 2000, 10733, 12570, 21713, 4487, 15474, 1012, 7
# ]
# }
# old_pv_turn_1 = {}
turn_2 = {
'user':
[13, 1045, 2215, 2000, 2681, 2044, 2459, 1024, 2321, 1012, 7]
}
old_pv_turn_2 = {
'labels': [[
13, 1045, 2052, 2066, 1037, 10095, 2013, 3002, 2198, 1005,
1055, 2267, 2000, 10733, 12570, 21713, 4487, 15474, 1012, 7
]],
'resp': [
14, 1045, 2052, 2022, 3407, 2000, 2393, 2007, 2115, 5227, 1010,
2079, 2017, 2031, 1037, 2051, 2017, 2052, 2066, 2000, 2681,
2030, 7180, 2011, 1029, 8
],
'bspn': [
15, 43, 7688, 10733, 12570, 21713, 4487, 15474, 6712, 3002,
2198, 1005, 1055, 2267, 9
],
'db': [19, 24, 21, 20],
'aspn': [16, 43, 48, 2681, 7180, 10]
}
pv_turn = self.trainer.forward(turn=turn_2, old_pv_turn=old_pv_turn_2)
return pv_turn

51
modelscope/pipelines/nlp/space/dialog_generation_pipeline.py
View File

@@ -1,51 +0,0 @@
from typing import Any, Dict, Optional
from modelscope.models.nlp import DialogGenerationModel
from modelscope.preprocessors import DialogGenerationPreprocessor
from modelscope.utils.constant import Tasks
from ...base import Model, Tensor
from ...builder import PIPELINES
__all__ = ['DialogGenerationPipeline']
@PIPELINES.register_module(
Tasks.dialog_generation, module_name=r'space-generation')
class DialogGenerationPipeline(Model):
def __init__(self, model: DialogGenerationModel,
preprocessor: DialogGenerationPreprocessor, **kwargs):
"""use `model` and `preprocessor` to create a nlp text classification pipeline for prediction
Args:
model (SequenceClassificationModel): a model instance
preprocessor (SequenceClassificationPreprocessor): a preprocessor instance
"""
super().__init__(model=model, preprocessor=preprocessor, **kwargs)
self.model = model
self.tokenizer = preprocessor.tokenizer
def postprocess(self, inputs: Dict[str, Tensor]) -> Dict[str, str]:
"""process the prediction results
Args:
inputs (Dict[str, Any]): _description_
Returns:
Dict[str, str]: the prediction results
"""
vocab_size = len(self.tokenizer.vocab)
pred_list = inputs['predictions']
pred_ids = pred_list[0][0].cpu().numpy().tolist()
for j in range(len(pred_ids)):
if pred_ids[j] >= vocab_size:
pred_ids[j] = 100
pred = self.tokenizer.convert_ids_to_tokens(pred_ids)
pred_string = ''.join(pred).replace(
'##',
'').split('[SEP]')[0].replace('[CLS]',
'').replace('[SEP]',
'').replace('[UNK]', '')
return {'pred_string': pred_string}

50
modelscope/preprocessors/space/dialog_generation_preprocessor.py
View File

@@ -1,50 +0,0 @@
# Copyright (c) Alibaba, Inc. and its affiliates.
import os
import uuid
from typing import Any, Dict, Union
from modelscope.preprocessors.space.fields.gen_field import \
MultiWOZBPETextField
from modelscope.utils.config import Config
from modelscope.utils.constant import Fields, InputFields
from modelscope.utils.type_assert import type_assert
from ..base import Preprocessor
from ..builder import PREPROCESSORS
__all__ = ['DialogGenerationPreprocessor']
@PREPROCESSORS.register_module(Fields.nlp, module_name=r'space-generation')
class DialogGenerationPreprocessor(Preprocessor):
def __init__(self, model_dir: str, *args, **kwargs):
"""preprocess the data via the vocab.txt from the `model_dir` path
Args:
model_dir (str): model path
"""
super().__init__(*args, **kwargs)
self.model_dir: str = model_dir
self.config = Config.from_file(
os.path.join(self.model_dir, 'configuration.json'))
self.text_field = MultiWOZBPETextField(
self.model_dir, config=self.config)
@type_assert(object, str)
def __call__(self, data: str) -> Dict[str, Any]:
"""process the raw input data
Args:
data (str): a sentence
Example:
'you are so handsome.'
Returns:
Dict[str, Any]: the preprocessed data
"""
idx = self.text_field.get_ids(data)
return {'user_idx': idx}

120
tests/pipelines/nlp/test_dialog_generation.py
View File

@@ -1,120 +0,0 @@
# Copyright (c) Alibaba, Inc. and its affiliates.
import os
import os.path as osp
import tempfile
import unittest
from modelscope.models.nlp import DialogGenerationModel
from modelscope.pipelines import DialogGenerationPipeline, pipeline
from modelscope.preprocessors import DialogGenerationPreprocessor
def merge(info, result):
return info
class DialogGenerationTest(unittest.TestCase):
test_case = {
'sng0073': {
'goal': {
'taxi': {
'info': {
'leaveat': '17:15',
'destination': 'pizza hut fen ditton',
'departure': "saint john's college"
},
'reqt': ['car', 'phone'],
'fail_info': {}
}
},
'log': [{
'user':
"i would like a taxi from saint john 's college to pizza hut fen ditton .",
'user_delex':
'i would like a taxi from [value_departure] to [value_destination] .',
'resp':
'what time do you want to leave and what time do you want to arrive by ?',
'sys':
'what time do you want to leave and what time do you want to arrive by ?',
'pointer': '0,0,0,0,0,0',
'match': '',
'constraint':
"[taxi] destination pizza hut fen ditton departure saint john 's college",
'cons_delex': '[taxi] destination departure',
'sys_act': '[taxi] [request] leave arrive',
'turn_num': 0,
'turn_domain': '[taxi]'
}, {
'user': 'i want to leave after 17:15 .',
'user_delex': 'i want to leave after [value_leave] .',
'resp':
'booking completed ! your taxi will be [value_car] contact number is [value_phone]',
'sys':
'booking completed ! your taxi will be blue honda contact number is 07218068540',
'pointer': '0,0,0,0,0,0',
'match': '',
'constraint':
"[taxi] destination pizza hut fen ditton departure saint john 's college leave 17:15",
'cons_delex': '[taxi] destination departure leave',
'sys_act': '[taxi] [inform] car phone',
'turn_num': 1,
'turn_domain': '[taxi]'
}, {
'user': 'thank you for all the help ! i appreciate it .',
'user_delex': 'thank you for all the help ! i appreciate it .',
'resp':
'you are welcome . is there anything else i can help you with today ?',
'sys':
'you are welcome . is there anything else i can help you with today ?',
'pointer': '0,0,0,0,0,0',
'match': '',
'constraint':
"[taxi] destination pizza hut fen ditton departure saint john 's college leave 17:15",
'cons_delex': '[taxi] destination departure leave',
'sys_act': '[general] [reqmore]',
'turn_num': 2,
'turn_domain': '[general]'
}, {
'user': 'no , i am all set . have a nice day . bye .',
'user_delex': 'no , i am all set . have a nice day . bye .',
'resp': 'you too ! thank you',
'sys': 'you too ! thank you',
'pointer': '0,0,0,0,0,0',
'match': '',
'constraint':
"[taxi] destination pizza hut fen ditton departure saint john 's college leave 17:15",
'cons_delex': '[taxi] destination departure leave',
'sys_act': '[general] [bye]',
'turn_num': 3,
'turn_domain': '[general]'
}]
}
}
def test_run(self):
# modeldir = '/Users/yangliu/Desktop/space-dialog-generation'
#
# preprocessor = DialogGenerationPreprocessor(model_dir=modeldir)
# model = DialogGenerationModel(
# model_dir=modeldir,
# text_field=preprocessor.text_field,
# config=preprocessor.config)
# print(model.forward(None))
# pipeline = DialogGenerationPipeline(model=model, preprocessor=preprocessor)
#
# history_dialog_info = {}
# for step, item in enumerate(test_case['sng0073']['log']):
# user_question = item['user']
# print('user: {}'.format(user_question))
#
# # history_dialog_info = merge(history_dialog_info,
# # result) if step > 0 else {}
# result = pipeline(user_question, history=history_dialog_info)
# #
# # print('sys : {}'.format(result['pred_answer']))
print('test')
if __name__ == '__main__':
unittest.main()

0
tests/preprocessors/nlp/__init__.py
View File

24
tests/preprocessors/nlp/test_dialog_generation.py
View File

@@ -1,24 +0,0 @@
# Copyright (c) Alibaba, Inc. and its affiliates.
import unittest
from maas_lib.preprocessors import DialogGenerationPreprocessor
from maas_lib.utils.constant import Fields, InputFields
from maas_lib.utils.logger import get_logger
from tests.case.nlp.dialog_generation_case import test_case
logger = get_logger()
class DialogGenerationPreprocessorTest(unittest.TestCase):
def test_tokenize(self):
modeldir = '/Users/yangliu/Desktop/space-dialog-generation'
processor = DialogGenerationPreprocessor(model_dir=modeldir)
for item in test_case['sng0073']['log']:
print(processor(item['user']))
if __name__ == '__main__':
unittest.main()

25
tests/preprocessors/nlp/test_dialog_intent.py
View File

@@ -1,25 +0,0 @@
# Copyright (c) Alibaba, Inc. and its affiliates.
import unittest
from maas_lib.preprocessors import DialogIntentPreprocessor
from maas_lib.utils.constant import Fields, InputFields
from maas_lib.utils.logger import get_logger
from tests.case.nlp.dialog_intent_case import test_case
logger = get_logger()
class DialogGenerationPreprocessorTest(unittest.TestCase):
def test_tokenize(self):
modeldir = '/Users/yangliu/Desktop/space-dialog-intent'
processor = DialogIntentPreprocessor(model_dir=modeldir)
for item in test_case:
print(item)
print(processor(item))
if __name__ == '__main__':
unittest.main()