ok Merge branch 'master' of github.com:modelscope/modelscope into release/1.24

modelscope/__init__.py

@@ -1,4 +1,5 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+import importlib
 from typing import TYPE_CHECKING
 
 from modelscope.utils.import_utils import (LazyImportModule,
@@ -109,6 +110,7 @@ else:
     }
 
     from modelscope.utils import hf_util
+    from modelscope.utils.hf_util.patcher import _patch_pretrained_class
 
     extra_objects = {}
     attributes = dir(hf_util)
@@ -116,6 +118,24 @@ else:
     for _import in imports:
         extra_objects[_import] = getattr(hf_util, _import)
 
+    def try_import_from_hf(name):
+        hf_pkgs = ['transformers', 'peft', 'diffusers']
+        module = None
+        for pkg in hf_pkgs:
+            try:
+                module = getattr(importlib.import_module(pkg), name)
+                break
+            except Exception:  # noqa
+                pass
+
+        if module is not None:
+            module = _patch_pretrained_class([module], wrap=True)
+        else:
+            raise ImportError(
+                f'Cannot import available module of {name} in modelscope,'
+                f' or related packages({hf_pkgs})')
+        return module[0]
+
     import sys
 
     sys.modules[__name__] = LazyImportModule(
@@ -124,4 +144,5 @@ else:
         _import_structure,
         module_spec=__spec__,
         extra_objects=extra_objects,
+        extra_import_func=try_import_from_hf,
     )

modelscope/msdatasets/ms_dataset.py

@@ -217,7 +217,7 @@ class MsDataset:
         download_mode = DownloadMode(download_mode
                                      or DownloadMode.REUSE_DATASET_IF_EXISTS)
         hub = Hubs(hub or Hubs.modelscope)
-
+        is_huggingface_hub = (hub == Hubs.huggingface)
         if not isinstance(dataset_name, str) and not isinstance(
                 dataset_name, list):
             raise TypeError(
@@ -233,7 +233,7 @@ class MsDataset:
         dataset_name = os.path.expanduser(dataset_name)
         is_local_path = os.path.exists(dataset_name)
         if is_relative_path(dataset_name) and dataset_name.count(
-                '/') == 1 and not is_local_path:
+                '/') == 1 and not is_local_path and not is_huggingface_hub:
             dataset_name_split = dataset_name.split('/')
             namespace = dataset_name_split[0].strip()
             dataset_name = dataset_name_split[1].strip()

modelscope/preprocessors/templates/loader.py

@@ -197,6 +197,13 @@ template_info = [
         modelfile_prefix=
         'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/codegemma',
     ),
+    TemplateInfo(
+        template=TemplateType.gemma,
+        template_regex=
+        f'{no("pali")}.*{cases("gemma3", "gemma-3")}.*',
+        modelfile_prefix=
+        'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/gemma3',
+    ),
     TemplateInfo(
         template=TemplateType.gemma,
         template_regex=
@@ -496,6 +503,13 @@ template_info = [
         'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/command-r',
     ),
 
+    TemplateInfo(
+        template_regex=
+        f'.*{cases("command-a")}.*',
+        modelfile_prefix=
+        'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/command-a',
+    ),
+
     # codegeex
     TemplateInfo(
         template=TemplateType.codegeex4,

modelscope/utils/automodel_utils.py

@@ -78,11 +78,10 @@ def get_default_automodel(config) -> Optional[type]:
 
 def get_hf_automodel_class(model_dir: str,
                            task_name: Optional[str]) -> Optional[type]:
-    from modelscope.utils.hf_util import (AutoConfig, AutoModel,
-                                          AutoModelForCausalLM,
-                                          AutoModelForSeq2SeqLM,
-                                          AutoModelForTokenClassification,
-                                          AutoModelForSequenceClassification)
+    from modelscope import (AutoConfig, AutoModel, AutoModelForCausalLM,
+                            AutoModelForSeq2SeqLM,
+                            AutoModelForTokenClassification,
+                            AutoModelForSequenceClassification)
     automodel_mapping = {
         Tasks.backbone: AutoModel,
         Tasks.chat: AutoModelForCausalLM,

modelscope/utils/hf_util/auto_class.py

@@ -73,14 +73,4 @@ if TYPE_CHECKING:
         AutoModelForKeypointDetection = None
 
 else:
-
-    from .patcher import get_all_imported_modules, _patch_pretrained_class
-    try:
-        all_available_modules = _patch_pretrained_class(
-            get_all_imported_modules(), wrap=True)
-    except Exception:  # noqa
-        import traceback
-        traceback.print_exc()
-    else:
-        for module in all_available_modules:
-            globals()[module.__name__] = module
+    pass

modelscope/utils/hf_util/patcher.py

@@ -25,10 +25,19 @@ def get_all_imported_modules():
     """Find all modules in transformers/peft/diffusers"""
     all_imported_modules = []
     transformers_include_names = [
-        'Auto.*', 'T5.*', 'BitsAndBytesConfig', 'GenerationConfig', 'Awq.*',
-        'GPTQ.*', 'BatchFeature', 'Qwen.*', 'Llama.*', 'PretrainedConfig',
-        'PreTrainedTokenizer', 'PreTrainedModel', 'PreTrainedTokenizerFast',
-        'Pipeline'
+        'Auto.*',
+        'T5.*',
+        'BitsAndBytesConfig',
+        'GenerationConfig',
+        'Awq.*',
+        'GPTQ.*',
+        'BatchFeature',
+        'Qwen.*',
+        'Llama.*',
+        'PretrainedConfig',
+        'PreTrainedTokenizer',
+        'PreTrainedModel',
+        'PreTrainedTokenizerFast',
     ]
     peft_include_names = ['.*PeftModel.*', '.*Config']
     diffusers_include_names = [

modelscope/utils/import_utils.py

@@ -389,7 +389,8 @@ class LazyImportModule(ModuleType):
                  import_structure,
                  module_spec=None,
                  extra_objects=None,
-                 try_to_pre_import=False):
+                 try_to_pre_import=False,
+                 extra_import_func=None):
         super().__init__(name)
         self._modules = set(import_structure.keys())
         self._class_to_module = {}
@@ -405,6 +406,7 @@ class LazyImportModule(ModuleType):
         self._objects = {} if extra_objects is None else extra_objects
         self._name = name
         self._import_structure = import_structure
+        self._extra_import_func = extra_import_func
         if try_to_pre_import:
             self._try_to_import()
 
@@ -434,6 +436,11 @@ class LazyImportModule(ModuleType):
         elif name in self._class_to_module.keys():
             module = self._get_module(self._class_to_module[name])
             value = getattr(module, name)
+        elif self._extra_import_func is not None:
+            value = self._extra_import_func(name)
+            if value is None:
+                raise AttributeError(
+                    f'module {self.__name__} has no attribute {name}')
         else:
             raise AttributeError(
                 f'module {self.__name__} has no attribute {name}')

tests/pipelines/test_llm_pipeline.py

@@ -174,6 +174,18 @@ class LLMPipelineTest(unittest.TestCase):
         print('messages: ', pipe(self.messages_en, **self.gen_cfg))
         print('prompt: ', pipe(self.prompt_en, **self.gen_cfg))
 
+    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
+    def test_qwen2(self):
+        pipe = pipeline(
+            task='chat',
+            model='qwen/Qwen2-0.5B-Instruct',
+            model_revision='master',
+            llm_first=True,
+            torch_dtype='torch.float16',
+            device_map='auto')
+        print('messages: ', pipe(self.messages_en, **self.gen_cfg))
+        print('prompt: ', pipe(self.prompt_en, **self.gen_cfg))
+
     @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
     def test_llama2chat(self):
         pipe = pipeline(

tests/tools/test_to_ollama.py

@@ -122,6 +122,14 @@ class TestToOllama(unittest.TestCase):
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_check_template_type(self):
+        _test_check_tmpl_type(
+            'lmstudio-community/c4ai-command-a-03-2025-GGUF',
+            'command-a',
+            gguf_meta={'general.name': 'C4Ai Command A 03 2025'})
+        _test_check_tmpl_type(
+            'unsloth/gemma-3-27b-it-GGUF',
+            'gemma3',
+            gguf_meta={'general.name': 'Gemma-3-27B-It'})
         _test_check_tmpl_type(
             'DevQuasar/CohereForAI.c4ai-command-r7b-arabic-02-2025-GGUF',
             'command-r7b-arabic',