Fix gpt_neo decode

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/10985686
This commit is contained in:
yuze.zyz
2022-12-06 12:29:42 +08:00
parent 5fd3e7bb43
commit a0d2581028

View File

@@ -38,3 +38,14 @@ class SentencePiecePreprocessor(Preprocessor):
def __call__(self, data: str) -> torch.Tensor:
return torch.tensor(self.tokenizer.encode([data]), dtype=torch.long)
def decode(self, tokens, **kwargs):
"""Decode the tokens to real text.
Args:
tokens: The output tokens from model's `forward` and `generate`
Returns:
The actual text.
"""
return self.tokenizer.decode(tokens)