Add whitelist to iic and damo (#1720)

modelscope/models/base/base_model.py

@@ -15,7 +15,7 @@ from modelscope.utils.constant import DEFAULT_MODEL_REVISION, Invoke, ModelFile
 from modelscope.utils.device import verify_device
 from modelscope.utils.logger import get_logger
 from modelscope.utils.plugins import (filter_plugin_in_whitelist,
-                                      register_modelhub_repo,
+                                      is_trusted_group, register_modelhub_repo,
                                       register_plugins_repo)
 
 logger = get_logger()
@@ -137,7 +137,8 @@ class Model(ABC):
             kwargs.pop(Invoke.KEY)
         else:
             invoked_by = Invoke.PRETRAINED
-
+        trust_remote_code = trust_remote_code or is_trusted_group(
+            model_name_or_path)
         ignore_file_pattern = kwargs.pop('ignore_file_pattern', None)
         if osp.exists(model_name_or_path):
             local_model_dir = model_name_or_path

modelscope/pipelines/builder.py

@@ -13,7 +13,7 @@ from modelscope.utils.hub import read_config
 from modelscope.utils.import_utils import is_transformers_available
 from modelscope.utils.logger import get_logger
 from modelscope.utils.plugins import (filter_plugin_in_whitelist,
-                                      register_modelhub_repo,
+                                      is_trusted_group, register_modelhub_repo,
                                       register_plugins_repo)
 from modelscope.utils.registry import Registry, build_from_cfg
 from modelscope.utils.task_utils import is_embedding_task
@@ -117,6 +117,9 @@ def pipeline(task: str = None,
     if task is None and pipeline_name is None:
         raise ValueError('task or pipeline_name is required')
 
+    model_id = model[0] if isinstance(model,
+                                      list) and len(model) > 0 else model
+    trust_remote_code = trust_remote_code or is_trusted_group(model_id)
     pipeline_props = None
     if pipeline_name is None:
         # get default pipeline for this task

modelscope/preprocessors/base.py

@@ -208,6 +208,7 @@ class Preprocessor(ABC):
                         revision: Optional[str] = DEFAULT_MODEL_REVISION,
                         cfg_dict: Config = None,
                         preprocessor_mode=ModeKeys.INFERENCE,
+                        trust_remote_code=False,
                         **kwargs):
         """Instantiate a preprocessor from local directory or remote model repo. Note
         that when loading from remote, the model revision can be specified.

modelscope/trainers/builder.py

@@ -5,7 +5,7 @@ from modelscope.pipelines.util import is_official_hub_path
 from modelscope.utils.constant import DEFAULT_MODEL_REVISION
 from modelscope.utils.hub import read_config
 from modelscope.utils.plugins import (filter_plugin_in_whitelist,
-                                      register_modelhub_repo,
+                                      is_trusted_group, register_modelhub_repo,
                                       register_plugins_repo)
 from modelscope.utils.registry import Registry, build_from_cfg
 
@@ -27,7 +27,10 @@ def build_trainer(name: str = Trainers.default, default_args: dict = None):
     default_args = default_args or {}
     model = default_args.get('model', None)
     model_revision = default_args.get('model_revision', DEFAULT_MODEL_REVISION)
-    trust_remote_code = default_args.get('trust_remote_code', False)
+    model_id = model[0] if isinstance(model,
+                                      list) and len(model) > 0 else model
+    trust_remote_code = default_args.get('trust_remote_code',
+                                         False) or is_trusted_group(model_id)
 
     if isinstance(model, str) \
             or (isinstance(model, list) and isinstance(model[0], str)):

modelscope/utils/plugins.py

@@ -61,6 +61,17 @@ def filter_plugin_in_whitelist(plugins):
     ]
 
 
+def is_trusted_group(model_id: Optional[str] = None) -> bool:
+    if not isinstance(model_id, str) or os.path.exists(model_id):
+        return False
+    pattern = re.compile(r'^([^/]+)/[^/]+$')
+    m = pattern.match(model_id)
+    if not m:
+        return False
+    group = m.group(1)
+    return group in ['iic', 'damo', 'modelscope']
+
+
 @contextmanager
 def pushd(new_dir: str, verbose: bool = False):
     """