visibility

modelscope/hub/api.py

@@ -39,7 +39,8 @@ from modelscope.hub.constants import (API_HTTP_CLIENT_MAX_RETRIES,
                                       MODELSCOPE_REQUEST_ID, ONE_YEAR_SECONDS,
                                       REQUESTS_API_HTTP_METHOD,
                                       TEMPORARY_FOLDER_NAME, DatasetVisibility,
-                                      Licenses, ModelVisibility)
+                                      Licenses, ModelVisibility, Visibility,
+                                      VisibilityMap)
 from modelscope.hub.errors import (InvalidParameter, NotExistError,
                                    NotLoginException, RequestError,
                                    datahub_raise_on_error,
@@ -59,9 +60,9 @@ from modelscope.utils.constant import (DEFAULT_DATASET_REVISION,
                                        REPO_TYPE_DATASET, REPO_TYPE_MODEL,
                                        REPO_TYPE_SUPPORT, ConfigFields,
                                        DatasetFormations, DatasetMetaFormats,
-                                       DatasetVisibilityMap, DownloadChannel,
-                                       DownloadMode, Frameworks, ModelFile,
-                                       Tasks, VirgoDatasetConfig)
+                                       DownloadChannel, DownloadMode,
+                                       Frameworks, ModelFile, Tasks,
+                                       VirgoDatasetConfig)
 from modelscope.utils.file_utils import get_file_hash, get_file_size
 from modelscope.utils.logger import get_logger
 from modelscope.utils.repo_utils import (DATASET_LFS_SUFFIX,
@@ -1095,7 +1096,7 @@ class HubApi:
         # get visibility of the dataset
         raise_on_error(resp)
         data = resp['Data']
-        visibility = DatasetVisibilityMap.get(data['Visibility'])
+        visibility = VisibilityMap.get(data['Visibility'])
 
         datahub_sts_url = f'{datahub_url}/ststoken?Revision={revision}'
         r_sts = self.session.get(url=datahub_sts_url, cookies=cookies,
@@ -1201,7 +1202,7 @@ class HubApi:
             repo_id: str,
             *,
             token: Union[str, bool, None] = None,
-            visibility: Optional[str] = 'public',
+            visibility: Optional[str] = Visibility.PUBLIC,
             repo_type: Optional[str] = REPO_TYPE_MODEL,
             chinese_name: Optional[str] = '',
             license: Optional[str] = Licenses.APACHE_V2,

modelscope/hub/constants.py

@@ -58,3 +58,16 @@ class DatasetVisibility(object):
     PRIVATE = 1
     INTERNAL = 3
     PUBLIC = 5
+
+
+class Visibility(object):
+    PRIVATE = 'private'
+    INTERNAL = 'internal'
+    PUBLIC = 'public'
+
+
+VisibilityMap = {
+    ModelVisibility.PRIVATE: Visibility.PRIVATE,
+    ModelVisibility.INTERNAL: Visibility.INTERNAL,
+    ModelVisibility.PUBLIC: Visibility.PUBLIC
+}

modelscope/hub/utils/utils.py

@@ -2,20 +2,15 @@
 
 import hashlib
 import os
-import shutil
-import tempfile
 from datetime import datetime
 from pathlib import Path
-from typing import BinaryIO, List, Optional, Union
-
-import requests
+from typing import List, Optional, Union
 
 from modelscope.hub.constants import (DEFAULT_MODELSCOPE_DOMAIN,
                                       DEFAULT_MODELSCOPE_GROUP,
                                       MODEL_ID_SEPARATOR, MODELSCOPE_SDK_DEBUG,
                                       MODELSCOPE_URL_SCHEME)
 from modelscope.hub.errors import FileIntegrityError
-from modelscope.utils.file_utils import get_default_modelscope_cache_dir
 from modelscope.utils.logger import get_logger
 
 logger = get_logger()

modelscope/msdatasets/utils/hf_datasets_util.py

@@ -59,6 +59,7 @@ from huggingface_hub.hf_api import HfApi, RepoFile, RepoFolder
 from packaging import version
 
 from modelscope import HubApi
+from modelscope.hub.constants import Visibility
 from modelscope.hub.utils.utils import get_endpoint
 from modelscope.msdatasets.utils.hf_file_utils import get_from_cache_ms
 from modelscope.utils.config_ds import MS_DATASETS_CACHE
@@ -724,7 +725,7 @@ def _download_additional_modules(
                 f"Please change relative import {import_name} to another name and add a '# From: URL_OR_PATH' "
                 f'comment pointing to the original relative import file path.'
             )
-        if import_type == 'internal':
+        if import_type == Visibility.INTERNAL:
             _api = HubApi()
             # url_or_filename = url_or_path_join(base_path, import_path + ".py")
             file_name = import_path + '.py'

modelscope/utils/constant.py

@@ -584,9 +584,6 @@ class MetaDataFields:
     ARGS_BIG_DATA = 'big_data'
 
 
-DatasetVisibilityMap = {1: 'private', 3: 'internal', 5: 'public'}
-
-
 class DistributedParallelType(object):
     """Parallel Strategies for Distributed Models"""
     DP = 'data_parallel'

modelscope/utils/hf_util/patcher.py

@@ -11,7 +11,8 @@ from pathlib import Path
 from types import MethodType
 from typing import BinaryIO, Dict, Iterable, List, Optional, Union
 
-from modelscope.hub.constants import DEFAULT_MODELSCOPE_DATA_ENDPOINT
+from modelscope.hub.constants import (DEFAULT_MODELSCOPE_DATA_ENDPOINT,
+                                      Visibility)
 from modelscope.utils.repo_utils import (CommitInfo, CommitOperation,
                                          CommitOperationAdd)
 
@@ -410,7 +411,7 @@ def _patch_hub():
         """
         from modelscope.hub.api import HubApi
         api = HubApi()
-        visibility = 'private' if private else 'public'
+        visibility = Visibility.PRIVATE if private else Visibility.PUBLIC
         repo_url = api.create_repo(
             repo_id, token=token, visibility=visibility, **kwargs)
         from modelscope.utils.repo_utils import RepoUrl