Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-09-02 03:59:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix encoding for Windows (#712)

Browse Source 
he proposed fix involves converting the encoding format from Windows-1250 to utf-8.

This is in response to the issues reported when running Anytext:

tyxsspa/AnyText#45
tyxsspa/AnyText#36
tyxsspa/AnyText#22
This commit is contained in:
Allan Kouidri
2024-01-09 14:16:04 +01:00
committed by GitHub
parent 105247140c
commit 383a4dc44f
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
modelscope/pipelines/nlp/translation_pipeline.py
View File

@@ -52,12 +52,12 @@ class TranslationPipeline(Pipeline):
self._src_vocab_path = osp.join(
model, self.cfg['dataset']['src_vocab']['file'])
self._src_vocab = dict([
(w.strip(), i) for i, w in enumerate(open(self._src_vocab_path))
(w.strip(), i) for i, w in enumerate(open(self._src_vocab_path, encoding='utf-8'))
])
self._trg_vocab_path = osp.join(
model, self.cfg['dataset']['trg_vocab']['file'])
self._trg_rvocab = dict([
(i, w.strip()) for i, w in enumerate(open(self._trg_vocab_path))
(i, w.strip()) for i, w in enumerate(open(self._trg_vocab_path, encoding='utf-8'))
])
tf_config = tf.ConfigProto(allow_soft_placement=True)
@@ -81,7 +81,7 @@ class TranslationPipeline(Pipeline):
self._tok = MosesTokenizer(lang=self._src_lang)
self._detok = MosesDetokenizer(lang=self._tgt_lang)
self._bpe = apply_bpe.BPE(open(self._src_bpe_path))
self._bpe = apply_bpe.BPE(open(self._src_bpe_path, encoding='utf-8'))
# model
output = self.model(self.input_wids)