diff --git a/modelscope/preprocessors/nlp/sentence_piece_preprocessor.py b/modelscope/preprocessors/nlp/sentence_piece_preprocessor.py index 6b0b76e1..fbaa7ace 100644 --- a/modelscope/preprocessors/nlp/sentence_piece_preprocessor.py +++ b/modelscope/preprocessors/nlp/sentence_piece_preprocessor.py @@ -38,3 +38,14 @@ class SentencePiecePreprocessor(Preprocessor): def __call__(self, data: str) -> torch.Tensor: return torch.tensor(self.tokenizer.encode([data]), dtype=torch.long) + + def decode(self, tokens, **kwargs): + """Decode the tokens to real text. + + Args: + tokens: The output tokens from model's `forward` and `generate` + + Returns: + The actual text. + """ + return self.tokenizer.decode(tokens)