diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 88875bfc..193b14a6 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -39,7 +39,8 @@ from modelscope.hub.constants import (API_HTTP_CLIENT_MAX_RETRIES, MODELSCOPE_REQUEST_ID, ONE_YEAR_SECONDS, REQUESTS_API_HTTP_METHOD, TEMPORARY_FOLDER_NAME, DatasetVisibility, - Licenses, ModelVisibility) + Licenses, ModelVisibility, Visibility, + VisibilityMap) from modelscope.hub.errors import (InvalidParameter, NotExistError, NotLoginException, RequestError, datahub_raise_on_error, @@ -59,9 +60,9 @@ from modelscope.utils.constant import (DEFAULT_DATASET_REVISION, REPO_TYPE_DATASET, REPO_TYPE_MODEL, REPO_TYPE_SUPPORT, ConfigFields, DatasetFormations, DatasetMetaFormats, - DatasetVisibilityMap, DownloadChannel, - DownloadMode, Frameworks, ModelFile, - Tasks, VirgoDatasetConfig) + DownloadChannel, DownloadMode, + Frameworks, ModelFile, Tasks, + VirgoDatasetConfig) from modelscope.utils.file_utils import get_file_hash, get_file_size from modelscope.utils.logger import get_logger from modelscope.utils.repo_utils import (DATASET_LFS_SUFFIX, @@ -1095,7 +1096,7 @@ class HubApi: # get visibility of the dataset raise_on_error(resp) data = resp['Data'] - visibility = DatasetVisibilityMap.get(data['Visibility']) + visibility = VisibilityMap.get(data['Visibility']) datahub_sts_url = f'{datahub_url}/ststoken?Revision={revision}' r_sts = self.session.get(url=datahub_sts_url, cookies=cookies, @@ -1201,7 +1202,7 @@ class HubApi: repo_id: str, *, token: Union[str, bool, None] = None, - visibility: Optional[str] = 'public', + visibility: Optional[str] = Visibility.PUBLIC, repo_type: Optional[str] = REPO_TYPE_MODEL, chinese_name: Optional[str] = '', license: Optional[str] = Licenses.APACHE_V2, diff --git a/modelscope/hub/constants.py b/modelscope/hub/constants.py index 2ed86a41..64b517c0 100644 --- a/modelscope/hub/constants.py +++ b/modelscope/hub/constants.py @@ -58,3 +58,16 @@ class DatasetVisibility(object): PRIVATE = 1 INTERNAL = 3 PUBLIC = 5 + + +class Visibility(object): + PRIVATE = 'private' + INTERNAL = 'internal' + PUBLIC = 'public' + + +VisibilityMap = { + ModelVisibility.PRIVATE: Visibility.PRIVATE, + ModelVisibility.INTERNAL: Visibility.INTERNAL, + ModelVisibility.PUBLIC: Visibility.PUBLIC +} diff --git a/modelscope/hub/utils/utils.py b/modelscope/hub/utils/utils.py index 3ad96fe2..7d377013 100644 --- a/modelscope/hub/utils/utils.py +++ b/modelscope/hub/utils/utils.py @@ -2,20 +2,15 @@ import hashlib import os -import shutil -import tempfile from datetime import datetime from pathlib import Path -from typing import BinaryIO, List, Optional, Union - -import requests +from typing import List, Optional, Union from modelscope.hub.constants import (DEFAULT_MODELSCOPE_DOMAIN, DEFAULT_MODELSCOPE_GROUP, MODEL_ID_SEPARATOR, MODELSCOPE_SDK_DEBUG, MODELSCOPE_URL_SCHEME) from modelscope.hub.errors import FileIntegrityError -from modelscope.utils.file_utils import get_default_modelscope_cache_dir from modelscope.utils.logger import get_logger logger = get_logger() diff --git a/modelscope/msdatasets/utils/hf_datasets_util.py b/modelscope/msdatasets/utils/hf_datasets_util.py index fea304f6..b1b3bc88 100644 --- a/modelscope/msdatasets/utils/hf_datasets_util.py +++ b/modelscope/msdatasets/utils/hf_datasets_util.py @@ -59,6 +59,7 @@ from huggingface_hub.hf_api import HfApi, RepoFile, RepoFolder from packaging import version from modelscope import HubApi +from modelscope.hub.constants import Visibility from modelscope.hub.utils.utils import get_endpoint from modelscope.msdatasets.utils.hf_file_utils import get_from_cache_ms from modelscope.utils.config_ds import MS_DATASETS_CACHE @@ -724,7 +725,7 @@ def _download_additional_modules( f"Please change relative import {import_name} to another name and add a '# From: URL_OR_PATH' " f'comment pointing to the original relative import file path.' ) - if import_type == 'internal': + if import_type == Visibility.INTERNAL: _api = HubApi() # url_or_filename = url_or_path_join(base_path, import_path + ".py") file_name = import_path + '.py' diff --git a/modelscope/utils/constant.py b/modelscope/utils/constant.py index ffc6f816..dbaffd1e 100644 --- a/modelscope/utils/constant.py +++ b/modelscope/utils/constant.py @@ -584,9 +584,6 @@ class MetaDataFields: ARGS_BIG_DATA = 'big_data' -DatasetVisibilityMap = {1: 'private', 3: 'internal', 5: 'public'} - - class DistributedParallelType(object): """Parallel Strategies for Distributed Models""" DP = 'data_parallel' diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py index 74264c13..787d1ef3 100644 --- a/modelscope/utils/hf_util/patcher.py +++ b/modelscope/utils/hf_util/patcher.py @@ -11,7 +11,8 @@ from pathlib import Path from types import MethodType from typing import BinaryIO, Dict, Iterable, List, Optional, Union -from modelscope.hub.constants import DEFAULT_MODELSCOPE_DATA_ENDPOINT +from modelscope.hub.constants import (DEFAULT_MODELSCOPE_DATA_ENDPOINT, + Visibility) from modelscope.utils.repo_utils import (CommitInfo, CommitOperation, CommitOperationAdd) @@ -410,7 +411,7 @@ def _patch_hub(): """ from modelscope.hub.api import HubApi api = HubApi() - visibility = 'private' if private else 'public' + visibility = Visibility.PRIVATE if private else Visibility.PUBLIC repo_url = api.create_repo( repo_id, token=token, visibility=visibility, **kwargs) from modelscope.utils.repo_utils import RepoUrl