From a0d25810289928072d91076ccd3dd94627d71e04 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 6 Dec 2022 12:29:42 +0800 Subject: [PATCH] Fix gpt_neo decode Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/10985686 --- .../preprocessors/nlp/sentence_piece_preprocessor.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/modelscope/preprocessors/nlp/sentence_piece_preprocessor.py b/modelscope/preprocessors/nlp/sentence_piece_preprocessor.py index 6b0b76e1..fbaa7ace 100644 --- a/modelscope/preprocessors/nlp/sentence_piece_preprocessor.py +++ b/modelscope/preprocessors/nlp/sentence_piece_preprocessor.py @@ -38,3 +38,14 @@ class SentencePiecePreprocessor(Preprocessor): def __call__(self, data: str) -> torch.Tensor: return torch.tensor(self.tokenizer.encode([data]), dtype=torch.long) + + def decode(self, tokens, **kwargs): + """Decode the tokens to real text. + + Args: + tokens: The output tokens from model's `forward` and `generate` + + Returns: + The actual text. + """ + return self.tokenizer.decode(tokens)