Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-02-25 04:30:48 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix comments issue

Browse Source
This commit is contained in:
mulin.lyh
2024-05-24 14:48:29 +08:00
parent 9b93c606c6
commit 7c649d4e32
4 changed files with 34 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
modelscope/cli/download.py
View File

@@ -43,15 +43,14 @@ class DownloadCMD(CLICommand):
'--local_dir',
type=str,
default=None,
help='The files in the library will be downloaded'
'directly in this directory without model id.'
'If this parameter provided cache_dir will be ignored.')
help='File will be downloaded to local location specified by'
'local_dir, in this case, cache_dir parameter will be ignored.')
parser.add_argument(
'files',
type=str,
default=None,
nargs='*',
help='Specify file paths in the repository to download'
help='Specify relative path to the repository file(s) to download.'
"(e.g 'tokenizer.json', 'onnx/decoder_model.onnx').")
parser.add_argument(
'--include',

2
modelscope/cli/login.py
View File

@@ -20,7 +20,7 @@ class LoginCMD(CLICommand):
@staticmethod
def define_args(parsers: ArgumentParser):
""" define args for download command.
""" define args for login command.
"""
parser = parsers.add_parser(LoginCMD.name)
parser.add_argument(

40
modelscope/hub/file_download.py
View File

@@ -56,7 +56,7 @@ def model_file_download(
local_files_only (bool, optional): If `True`, avoid downloading the file and return the path to the
local cached file if it exists. if `False`, download the file anyway even it exists.
cookies (CookieJar, optional): The cookie of download request.
local_dir (str, optional): Specific download file location.
local_dir (str, optional): Specific local directory path to which the file will be downloaded.
Returns:
string: string of local file or if networking is off, last version of
@@ -76,22 +76,8 @@ def model_file_download(
- [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
if some parameter value is invalid
"""
group_or_owner, name = model_id_to_group_owner_name(model_id)
if local_dir is not None:
temporary_cache_dir = os.path.join(local_dir, TEMPORARY_FOLDER_NAME)
cache = ModelFileSystemCache(local_dir)
else:
if cache_dir is None:
cache_dir = get_model_cache_root()
if isinstance(cache_dir, Path):
cache_dir = str(cache_dir)
temporary_cache_dir = os.path.join(cache_dir, TEMPORARY_FOLDER_NAME,
group_or_owner, name)
name = name.replace('.', '___')
cache = ModelFileSystemCache(cache_dir, group_or_owner, name)
os.makedirs(temporary_cache_dir, exist_ok=True)
temporary_cache_dir, cache = create_temporary_directory_and_cache(
model_id, local_dir, cache_dir)
# if local_files_only is `True` and the file already exists in cached_path
# return the cached path
if local_files_only:
@@ -173,6 +159,26 @@ def model_file_download(
os.path.join(temporary_cache_dir, file_path))
def create_temporary_directory_and_cache(model_id: str, local_dir: str,
cache_dir: str):
group_or_owner, name = model_id_to_group_owner_name(model_id)
if local_dir is not None:
temporary_cache_dir = os.path.join(local_dir, TEMPORARY_FOLDER_NAME)
cache = ModelFileSystemCache(local_dir)
else:
if cache_dir is None:
cache_dir = get_model_cache_root()
if isinstance(cache_dir, Path):
cache_dir = str(cache_dir)
temporary_cache_dir = os.path.join(cache_dir, TEMPORARY_FOLDER_NAME,
group_or_owner, name)
name = name.replace('.', '___')
cache = ModelFileSystemCache(cache_dir, group_or_owner, name)
os.makedirs(temporary_cache_dir, exist_ok=True)
return temporary_cache_dir, cache
def get_file_download_url(model_id: str, file_path: str, revision: str):
"""Format file download url according to `model_id`, `revision` and `file_path`.
e.g., Given `model_id=john/bert`, `revision=master`, `file_path=README.md`,

30
modelscope/hub/snapshot_download.py
View File

@@ -8,16 +8,13 @@ from typing import Dict, List, Optional, Union
from modelscope.hub.api import HubApi, ModelScopeConfig
from modelscope.utils.constant import DEFAULT_MODEL_REVISION
from modelscope.utils.file_utils import get_model_cache_root
from modelscope.utils.logger import get_logger
from .constants import (FILE_HASH, MODELSCOPE_DOWNLOAD_PARALLELS,
MODELSCOPE_PARALLEL_DOWNLOAD_THRESHOLD_MB,
TEMPORARY_FOLDER_NAME)
from .file_download import (get_file_download_url, http_get_file,
MODELSCOPE_PARALLEL_DOWNLOAD_THRESHOLD_MB)
from .file_download import (create_temporary_directory_and_cache,
get_file_download_url, http_get_file,
parallel_download)
from .utils.caching import ModelFileSystemCache
from .utils.utils import (file_integrity_validation,
model_id_to_group_owner_name)
from .utils.utils import file_integrity_validation
logger = get_logger()
@@ -55,7 +52,7 @@ def snapshot_download(
Any file pattern to be ignored in downloading, like exact file names or file extensions.
allow_file_pattern (`str` or `List`, *optional*, default to `None`):
Any file pattern to be downloading, like exact file names or file extensions.
local_dir (str, optional): Specific download file location.
local_dir (str, optional): Specific local directory path to which the file will be downloaded.
Raises:
ValueError: the value details.
@@ -71,21 +68,8 @@ def snapshot_download(
- [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
if some parameter value is invalid
"""
group_or_owner, name = model_id_to_group_owner_name(model_id)
if local_dir is not None:
temporary_cache_dir = os.path.join(local_dir, TEMPORARY_FOLDER_NAME)
cache = ModelFileSystemCache(local_dir)
else:
if cache_dir is None:
cache_dir = get_model_cache_root()
if isinstance(cache_dir, Path):
cache_dir = str(cache_dir)
temporary_cache_dir = os.path.join(cache_dir, TEMPORARY_FOLDER_NAME,
group_or_owner, name)
name = name.replace('.', '___')
cache = ModelFileSystemCache(cache_dir, group_or_owner, name)
os.makedirs(temporary_cache_dir, exist_ok=True)
temporary_cache_dir, cache = create_temporary_directory_and_cache(
model_id, local_dir, cache_dir)
if local_files_only:
if len(cache.cached_files) == 0: