From c67b2cfc342c2fc44f861c665cacdb539d1d09db Mon Sep 17 00:00:00 2001 From: "mulin.lyh" Date: Mon, 16 Oct 2023 22:12:31 +0800 Subject: [PATCH 1/6] fix ofa new transformers compatible issue Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/14317517 * fix ofa new transformers compatible issue * fix timm.layers to timm.models.layers compatible issue --- modelscope/models/cv/shop_segmentation/head_fpn.py | 4 ++-- modelscope/models/cv/shop_segmentation/models.py | 4 ++-- modelscope/models/cv/shop_segmentation/neck_fpn.py | 4 ++-- .../models/multi_modal/ofa/tokenization_ofa.py | 12 ++++++------ 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/modelscope/models/cv/shop_segmentation/head_fpn.py b/modelscope/models/cv/shop_segmentation/head_fpn.py index dfa284d4..0d4027cb 100644 --- a/modelscope/models/cv/shop_segmentation/head_fpn.py +++ b/modelscope/models/cv/shop_segmentation/head_fpn.py @@ -9,8 +9,8 @@ import numpy as np import torch import torch.nn as nn from mmcv.cnn import ConvModule -from timm.layers.drop import drop_path -from timm.layers.weight_init import trunc_normal_ +from timm.models.layers.drop import drop_path +from timm.models.layers.weight_init import trunc_normal_ from .common import Upsample, resize diff --git a/modelscope/models/cv/shop_segmentation/models.py b/modelscope/models/cv/shop_segmentation/models.py index 1b07a08c..a206e9f1 100644 --- a/modelscope/models/cv/shop_segmentation/models.py +++ b/modelscope/models/cv/shop_segmentation/models.py @@ -11,8 +11,8 @@ from collections import OrderedDict import torch import torch.nn.functional as F import torch.utils.checkpoint as checkpoint -from timm.layers.drop import drop_path -from timm.layers.weight_init import trunc_normal_ +from timm.models.layers.drop import drop_path +from timm.models.layers.weight_init import trunc_normal_ from torch import nn diff --git a/modelscope/models/cv/shop_segmentation/neck_fpn.py b/modelscope/models/cv/shop_segmentation/neck_fpn.py index 12c11d76..d344de71 100644 --- a/modelscope/models/cv/shop_segmentation/neck_fpn.py +++ b/modelscope/models/cv/shop_segmentation/neck_fpn.py @@ -8,8 +8,8 @@ import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule -from timm.layers.drop import drop_path -from timm.layers.weight_init import trunc_normal_ +from timm.models.layers.drop import drop_path +from timm.models.layers.weight_init import trunc_normal_ from .common import resize diff --git a/modelscope/models/multi_modal/ofa/tokenization_ofa.py b/modelscope/models/multi_modal/ofa/tokenization_ofa.py index 77de7a1d..ea79a327 100644 --- a/modelscope/models/multi_modal/ofa/tokenization_ofa.py +++ b/modelscope/models/multi_modal/ofa/tokenization_ofa.py @@ -183,6 +183,12 @@ class OFATokenizerZH(PreTrainedTokenizer): tokenize_chinese_chars=True, strip_accents=None, **kwargs): + if not os.path.isfile(vocab_file): + raise ValueError( + f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained " + 'model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`' + ) + self.vocab = load_vocab(vocab_file) super().__init__( do_lower_case=do_lower_case, do_basic_tokenize=do_basic_tokenize, @@ -199,12 +205,6 @@ class OFATokenizerZH(PreTrainedTokenizer): **kwargs, ) - if not os.path.isfile(vocab_file): - raise ValueError( - f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained " - 'model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`' - ) - self.vocab = load_vocab(vocab_file) self.ids_to_tokens = collections.OrderedDict([ (ids, tok) for tok, ids in self.vocab.items() ]) From c01141f97e1cf28c5f67a75e988513b4e01deffe Mon Sep 17 00:00:00 2001 From: Jintao Date: Thu, 12 Oct 2023 10:27:31 +0800 Subject: [PATCH 2/6] fix merge error (#582) --- modelscope/models/base/base_model.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/modelscope/models/base/base_model.py b/modelscope/models/base/base_model.py index a3b65812..8e6d4ae6 100644 --- a/modelscope/models/base/base_model.py +++ b/modelscope/models/base/base_model.py @@ -9,7 +9,7 @@ from modelscope.metainfo import Tasks from modelscope.models.builder import build_backbone, build_model from modelscope.utils.automodel_utils import (can_load_by_ms, try_to_load_hf_model) -from modelscope.utils.config import Config +from modelscope.utils.config import Config, ConfigDict from modelscope.utils.constant import DEFAULT_MODEL_REVISION, Invoke, ModelFile from modelscope.utils.device import verify_device from modelscope.utils.logger import get_logger @@ -142,15 +142,10 @@ class Model(ABC): task_name = cfg.task if 'task' in kwargs: task_name = kwargs.pop('task') - try: - model_cfg = cfg.model - if hasattr(model_cfg, - 'model_type') and not hasattr(model_cfg, 'type'): - model_cfg.type = model_cfg.model_type - model_type = model_cfg.type - except Exception: - model_cfg = {} - model_type = '' + model_cfg = getattr(cfg, 'model', ConfigDict()) + if hasattr(model_cfg, 'model_type') and not hasattr(model_cfg, 'type'): + model_cfg.type = model_cfg.model_type + model_type = getattr(model_cfg, 'type', None) if isinstance(device, str) and device.startswith('gpu'): device = 'cuda' + device[3:] use_hf = kwargs.pop('use_hf', None) @@ -162,7 +157,7 @@ class Model(ABC): model = try_to_load_hf_model(local_model_dir, task_name, use_hf, **kwargs) if model is not None: - device_map = kwargs.get('device_map', None) + device_map = kwargs.pop('device_map', None) if device_map is None and device is not None: model = model.to(device) return model From f5b83ebd83bf421e13295172213c06b589c8863f Mon Sep 17 00:00:00 2001 From: "mulin.lyh" Date: Tue, 17 Oct 2023 22:15:54 +0800 Subject: [PATCH 3/6] fix chatglm2 can't find tokenizer issue Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/14335080 * fix chatglm2 can't find tokenizer issue --- modelscope/models/nlp/chatglm2/tokenization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelscope/models/nlp/chatglm2/tokenization.py b/modelscope/models/nlp/chatglm2/tokenization.py index 7014dc9c..4523dcdd 100644 --- a/modelscope/models/nlp/chatglm2/tokenization.py +++ b/modelscope/models/nlp/chatglm2/tokenization.py @@ -72,7 +72,6 @@ class ChatGLM2Tokenizer(PreTrainedTokenizer): model_input_names = ['input_ids', 'attention_mask', 'position_ids'] def __init__(self, vocab_file, padding_side='left', **kwargs): - super().__init__(padding_side=padding_side, **kwargs) self.name = 'GLMTokenizer' self.vocab_file = vocab_file @@ -82,6 +81,7 @@ class ChatGLM2Tokenizer(PreTrainedTokenizer): '': self.tokenizer.eos_id, '': self.tokenizer.pad_id } + super().__init__(padding_side=padding_side, **kwargs) def get_command(self, token): if token in self.special_tokens: From f568454bbef187a9e8316f300bd061a340f5bc35 Mon Sep 17 00:00:00 2001 From: "suluyan.sly" Date: Wed, 18 Oct 2023 16:29:13 +0800 Subject: [PATCH 4/6] [swingdeploy] oss examples Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/14342415 * oss_examples --- modelscope/utils/pipeline_inputs.json | 66 +++++++++++++-------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/modelscope/utils/pipeline_inputs.json b/modelscope/utils/pipeline_inputs.json index 0cb9c1b1..03a00636 100644 --- a/modelscope/utils/pipeline_inputs.json +++ b/modelscope/utils/pipeline_inputs.json @@ -1,17 +1,17 @@ { "action-detection":{ "input":{ - "video":"data/test/videos/action_detection_test_video.mp4" + "video":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/videos/action_detection_test_video.mp4" } }, "action-recognition":{ "input":{ - "video":"data/test/videos/action_recognition_test_video.mp4" + "video":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/videos/action_recognition_test_video.mp4" } }, "animal-recognition":{ "input":{ - "image":"data/test/images/dogs.jpg" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/dogs.jpg" } }, "chat":{ @@ -34,52 +34,52 @@ }, "domain-specific-object-detection":{ "input":{ - "image":"data/test/images/image_traffic_sign.jpg" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/image_traffic_sign.jpg" } }, "face-2d-keypoints":{ "input":{ - "image":"data/test/images/face_detection.png" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/face_detection.png" } }, "face-attribute-recognition":{ "input":{ - "image":"data/test/images/face_recognition_1.png" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/face_recognition_1.png" } }, "facial-expression-recognition":{ "input":{ - "image":"data/test/images/facial_expression_recognition.jpg" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/facial_expression_recognition.jpg" } }, "general-recognition":{ "input":{ - "image":"data/test/images/dogs.jpg" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/dogs.jpg" } }, "human-detection":{ "input":{ - "image":"data/test/images/image_detection.jpg" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/image_detection.jpg" } }, "image-captioning":{ "input":{ - "image":"data/test/images/image_captioning.png" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/image_captioning.png" } }, "image-classification":{ "input":{ - "image":"data/test/images/content_check.jpg" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/content_check.jpg" } }, "image-demoireing":{ "input":{ - "image":"data/test/images/shop_segmentation.jpg" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/shop_segmentation.jpg" } }, "image-object-detection":{ "input":{ - "image":"data/test/images/image_detection.jpg" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/image_detection.jpg" } }, "image-portrait-stylization":{ @@ -89,7 +89,7 @@ }, "image-segmentation":{ "input":{ - "image":"data/test/images/image_semantic_segmentation.jpg" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/image_semantic_segmentation.jpg" }, "parameters":{ @@ -97,18 +97,18 @@ }, "image-text-retrieval":{ "input":{ - "image":"data/test/images/image_mplug_vqa.jpg", + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/image_mplug_vqa.jpg", "text":"What is the woman doing?" } }, "indoor-layout-estimation":{ "input":{ - "image":"data/test/images/image_traffic_sign.jpg" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/image_traffic_sign.jpg" } }, "live-category":{ "input":{ - "video":"data/test/videos/live_category_test_video.mp4" + "video":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/videos/live_category_test_video.mp4" } }, "motion-generation":{ @@ -132,22 +132,22 @@ }, "ocr-recognition":{ "input":{ - "image":"data/test/images/image_ocr_recognition.jpg" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/image_ocr_recognition.jpg" } }, "panorama-depth-estimation":{ "input":{ - "image":"data/test/images/panorama_depth_estimation.jpg" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/panorama_depth_estimation.jpg" } }, "semantic-segmentation":{ "input":{ - "image":"data/test/images/image_salient_detection.jpg" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/image_salient_detection.jpg" } }, "shop-segmentation":{ "input":{ - "image":"data/test/images/shop_segmentation.jpg" + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/shop_segmentation.jpg" } }, "text-classification":{ @@ -160,7 +160,7 @@ }, "text-driven-segmentation":{ "input":{ - "image":"data/test/images/text_driven_segmentation.jpg", + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/text_driven_segmentation.jpg", "text":"bear" } }, @@ -201,60 +201,60 @@ }, "video-captioning":{ "input":{ - "video":"data/test/videos/video_caption_and_qa_test.mp4" + "video":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/videos/video_caption_and_qa_test.mp4" } }, "video-category":{ "input":{ - "video":"data/test/videos/video_category_test_video.mp4" + "video":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/videos/video_category_test_video.mp4" } }, "video-depth-estimation":{ "input":{ - "video":"data/test/videos/video_depth_estimation.mp4" + "video":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/videos/video_depth_estimation.mp4" } }, "video-embedding":{ "input":{ - "video":"data/test/videos/action_recognition_test_video.mp4" + "video":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/videos/action_recognition_test_video.mp4" } }, "video-multi-object-tracking":{ "input":{ - "video":"data/test/videos/MOT17-03-partial.mp4" + "video":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/videos/MOT17-03-partial.mp4" } }, "video-panoptic-segmentation":{ "input":{ - "video":"data/test/videos/kitti-step_testing_image_02_0000.mp4" + "video":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/videos/kitti-step_testing_image_02_0000.mp4" } }, "video-question-answering":{ "input":{ - "video":"data/test/videos/video_caption_and_qa_test.mp4", + "video":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/videos/video_caption_and_qa_test.mp4", "text":"How many people are there?" } }, "video-summarization":{ "input":{ - "text":"data/test/videos/video_category_test_video.mp4" + "text":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/videos/video_category_test_video.mp4" } }, "visual-entailment":{ "input":{ - "image":"data/test/images/dogs.jpg", + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/dogs.jpg", "text":"there are two birds." } }, "visual-grounding":{ "input":{ - "image":"data/test/images/visual_grounding.png", + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/visual_grounding.png", "text":"a blue turtle-like pokemon with round head" } }, "visual-question-answering":{ "input":{ - "image":"data/test/images/image_mplug_vqa.jpg", + "image":"http://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/image_mplug_vqa.jpg", "text":"What is the woman doing?" } }, From 2c3bf9629d76d401cfe962106f72685771f1e416 Mon Sep 17 00:00:00 2001 From: "wenmeng.zwm" Date: Wed, 18 Oct 2023 20:24:42 +0800 Subject: [PATCH 5/6] fix chatglm sp_tokenizer error Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/14352495 --- modelscope/models/nlp/chatglm/tokenization.py | 7 ++++--- modelscope/models/nlp/llama/text_generation.py | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modelscope/models/nlp/chatglm/tokenization.py b/modelscope/models/nlp/chatglm/tokenization.py index f5f8cd0c..6ce1b90d 100644 --- a/modelscope/models/nlp/chatglm/tokenization.py +++ b/modelscope/models/nlp/chatglm/tokenization.py @@ -199,6 +199,10 @@ class ChatGLMTokenizer(PreTrainedTokenizer): padding_side='left', num_image_tokens=20000, **kwargs) -> None: + + self.sp_tokenizer = SPTokenizer( + vocab_file, num_image_tokens=num_image_tokens) + super().__init__( do_lower_case=do_lower_case, remove_space=remove_space, @@ -220,9 +224,6 @@ class ChatGLMTokenizer(PreTrainedTokenizer): self.end_token = end_token self.mask_token = mask_token self.gmask_token = gmask_token - - self.sp_tokenizer = SPTokenizer( - vocab_file, num_image_tokens=num_image_tokens) """ Initialisation """ @property diff --git a/modelscope/models/nlp/llama/text_generation.py b/modelscope/models/nlp/llama/text_generation.py index b9cc8032..d95cae34 100644 --- a/modelscope/models/nlp/llama/text_generation.py +++ b/modelscope/models/nlp/llama/text_generation.py @@ -71,6 +71,8 @@ def get_chat_prompt(system: str, text: str, history: List[Tuple[str, str]], # This file is mainly copied from the llama code of transformers +@MODELS.register_module(Tasks.chat, module_name=Models.llama2) +@MODELS.register_module(Tasks.chat, module_name=Models.llama) @MODELS.register_module(Tasks.text_generation, module_name=Models.llama2) @MODELS.register_module(Tasks.chat, module_name=Models.llama2) @MODELS.register_module(Tasks.text_generation, module_name=Models.llama) From 0390bdea676a1804ddd7099b6d8aef57a695fda2 Mon Sep 17 00:00:00 2001 From: "mulin.lyh" Date: Thu, 19 Oct 2023 12:12:35 +0800 Subject: [PATCH 6/6] remove llama2 dup in chat task Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/14359280 * remove llama2 dup in chat task --- modelscope/models/nlp/llama/text_generation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modelscope/models/nlp/llama/text_generation.py b/modelscope/models/nlp/llama/text_generation.py index d95cae34..45b9d5f0 100644 --- a/modelscope/models/nlp/llama/text_generation.py +++ b/modelscope/models/nlp/llama/text_generation.py @@ -74,7 +74,6 @@ def get_chat_prompt(system: str, text: str, history: List[Tuple[str, str]], @MODELS.register_module(Tasks.chat, module_name=Models.llama2) @MODELS.register_module(Tasks.chat, module_name=Models.llama) @MODELS.register_module(Tasks.text_generation, module_name=Models.llama2) -@MODELS.register_module(Tasks.chat, module_name=Models.llama2) @MODELS.register_module(Tasks.text_generation, module_name=Models.llama) class LlamaForTextGeneration(MsModelMixin, LlamaForCausalLM, TorchModel):