mirror of
https://github.com/modelscope/modelscope.git
synced 2025-12-25 12:39:25 +01:00
Fix gpt_neo decode
Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/10985686
This commit is contained in:
@@ -38,3 +38,14 @@ class SentencePiecePreprocessor(Preprocessor):
|
||||
|
||||
def __call__(self, data: str) -> torch.Tensor:
|
||||
return torch.tensor(self.tokenizer.encode([data]), dtype=torch.long)
|
||||
|
||||
def decode(self, tokens, **kwargs):
|
||||
"""Decode the tokens to real text.
|
||||
|
||||
Args:
|
||||
tokens: The output tokens from model's `forward` and `generate`
|
||||
|
||||
Returns:
|
||||
The actual text.
|
||||
"""
|
||||
return self.tokenizer.decode(tokens)
|
||||
|
||||
Reference in New Issue
Block a user