mirror of
https://github.com/modelscope/modelscope.git
synced 2026-02-24 04:01:10 +01:00
Merge remote-tracking branch 'origin/release/1.33' into build_swift_image
This commit is contained in:
120
docker/Dockerfile.ascend
Normal file
120
docker/Dockerfile.ascend
Normal file
@@ -0,0 +1,120 @@
|
||||
FROM {base_image}
|
||||
|
||||
RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
|
||||
pip config set install.trusted-host mirrors.aliyun.com
|
||||
|
||||
# Prepare required system dependencies
|
||||
RUN apt-get update -y && \
|
||||
apt-get install -y --no-install-recommends gcc g++ cmake libnuma-dev wget git curl jq vim build-essential && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
|
||||
pip install --upgrade pip setuptools packaging && \
|
||||
pip cache purge
|
||||
|
||||
{extra_content}
|
||||
|
||||
# Prepare repositories with low update frequency
|
||||
RUN ARCH=$(uname -m) && \
|
||||
# Set extra pip index for x86_64 platform
|
||||
echo "[LOG INFO] Detected architecture: $ARCH" && \
|
||||
if [ "$ARCH" = "x86_64" ]; then \
|
||||
pip config set global.extra-index-url "https://download.pytorch.org/whl/cpu/"; \
|
||||
fi && \
|
||||
# Clone libs
|
||||
git clone --depth 1 --branch v0.11.0 https://github.com/vllm-project/vllm && \
|
||||
git clone --depth 1 --branch v0.11.0rc1 https://github.com/vllm-project/vllm-ascend.git && \
|
||||
git clone https://gitcode.com/Ascend/MindSpeed.git && \
|
||||
cd MindSpeed && git checkout f2b0977e && cd .. && \
|
||||
git clone --depth 1 --branch core_v0.12.1 https://github.com/NVIDIA/Megatron-LM.git
|
||||
|
||||
# Install repositories with low update frequency
|
||||
RUN ARCH=$(uname -m) && \
|
||||
# Export and source env
|
||||
if [ "$ARCH" = "aarch64" ]; then \
|
||||
export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/8.3.RC1/aarch64-linux/devlib/linux/aarch64:$LD_LIBRARY_PATH; \
|
||||
elif [ "$ARCH" = "x86_64" ]; then \
|
||||
export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/8.3.RC1/x86_64-linux/devlib/linux/x86_64/:$LD_LIBRARY_PATH; \
|
||||
fi && \
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh && \
|
||||
# Install torch & torch_npu & torchvision
|
||||
pip install torch==2.7.1 torch_npu==2.7.1 torchvision==0.22.1 && \
|
||||
# Install vllm
|
||||
cd vllm && VLLM_TARGET_DEVICE=empty pip install -v -e . && cd .. && \
|
||||
# Install vllm-ascend
|
||||
cd vllm-ascend && pip install -v -e . && cd .. && \
|
||||
# Install MindSpeed & Megatron
|
||||
pip install -e MindSpeed && \
|
||||
# Clear extra files
|
||||
rm -rf /tmp/* /var/tmp/* && \
|
||||
pip cache purge
|
||||
|
||||
ENV PYTHONPATH ${PYTHONPATH}:/Megatron-LM
|
||||
|
||||
ARG INSTALL_MS_DEPS={install_ms_deps}
|
||||
|
||||
# install dependencies
|
||||
COPY requirements /var/modelscope
|
||||
|
||||
RUN pip uninstall ms-swift modelscope -y && pip --no-cache-dir install pip==23.* -U && \
|
||||
if [ "$INSTALL_MS_DEPS" = "True" ]; then \
|
||||
pip --no-cache-dir install omegaconf==2.0.6 && \
|
||||
pip install 'editdistance==0.8.1' && \
|
||||
pip install --no-cache-dir 'cython<=0.29.36' versioneer 'numpy<2.0' -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
|
||||
pip install --no-cache-dir -r /var/modelscope/framework.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
|
||||
pip install --no-cache-dir -r /var/modelscope/audio.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
|
||||
pip install --no-cache-dir -r /var/modelscope/cv.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
|
||||
pip install --no-cache-dir -r /var/modelscope/multi-modal.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
|
||||
pip install --no-cache-dir -r /var/modelscope/nlp.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
|
||||
pip install --no-cache-dir -r /var/modelscope/science.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
|
||||
pip install --no-cache-dir -r /var/modelscope/tests.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
|
||||
pip install --no-cache-dir -r /var/modelscope/server.txt && \
|
||||
pip install --no-cache-dir https://modelscope.oss-cn-beijing.aliyuncs.com/packages/imageio_ffmpeg-0.4.9-py3-none-any.whl --no-dependencies --force && \
|
||||
pip install adaseq pai-easycv && \
|
||||
pip install --no-cache-dir 'scipy<1.13.0' && \
|
||||
pip install --no-cache-dir funtextprocessing typeguard==2.13.3 scikit-learn -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
|
||||
pip install --no-cache-dir text2sql_lgesql==1.3.0 git+https://github.com/jin-s13/xtcocoapi.git@v1.14 git+https://github.com/gatagat/lap.git@v0.4.0 -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --force --no-deps && \
|
||||
pip install --no-cache-dir mmcls>=0.21.0 mmdet>=2.25.0 decord>=0.6.0 mpi4py paint_ldm ipykernel fasttext -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
|
||||
pip uninstall ddpm_guided_diffusion -y && \
|
||||
pip install --no-cache-dir 'blobfile>=1.0.5' && \
|
||||
pip install 'ddpm_guided_diffusion' -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --no-index && \
|
||||
pip uninstall shotdetect_scenedetect_lgss -y && \
|
||||
pip install 'shotdetect_scenedetect_lgss' -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --no-index && \
|
||||
pip uninstall MinDAEC -y && \
|
||||
pip install https://modelscope.oss-cn-beijing.aliyuncs.com/releases/dependencies/MinDAEC-0.0.2-py3-none-any.whl && \
|
||||
pip cache purge; \
|
||||
else \
|
||||
pip install --no-cache-dir -r /var/modelscope/framework.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
|
||||
pip cache purge; \
|
||||
fi
|
||||
|
||||
ARG CUR_TIME={cur_time}
|
||||
RUN echo $CUR_TIME
|
||||
|
||||
RUN curl -fsSL https://ollama.com/install.sh | sh && \
|
||||
pip install --no-cache-dir -U funasr scikit-learn && \
|
||||
pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa timm transformers accelerate peft trl safetensors && \
|
||||
cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {swift_branch} --single-branch https://github.com/modelscope/ms-swift.git && \
|
||||
cd ms-swift && pip install .[llm] && \
|
||||
pip install .[eval] && pip install evalscope -U --no-dependencies && pip install ms-agent -U --no-dependencies && \
|
||||
cd / && rm -fr /tmp/ms-swift && pip cache purge; \
|
||||
cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {modelscope_branch} --single-branch https://github.com/modelscope/modelscope.git && \
|
||||
cd modelscope && pip install . -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
|
||||
cd / && rm -fr /tmp/modelscope && pip cache purge; \
|
||||
pip install --no-cache-dir transformers diffusers timm>=0.9.0 && pip cache purge; \
|
||||
pip install --no-cache-dir omegaconf==2.3.0 && pip cache purge;
|
||||
|
||||
|
||||
RUN if [ "$INSTALL_MS_DEPS" = "True" ]; then \
|
||||
pip install --no-cache-dir huggingface-hub transformers peft -U; \
|
||||
fi;
|
||||
|
||||
ENV SETUPTOOLS_USE_DISTUTILS=stdlib
|
||||
ENV VLLM_USE_MODELSCOPE=True
|
||||
ENV LMDEPLOY_USE_MODELSCOPE=True
|
||||
ENV MODELSCOPE_CACHE=/mnt/workspace/.cache/modelscope/hub
|
||||
|
||||
# Show install results
|
||||
RUN pip list
|
||||
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
@@ -417,6 +417,38 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
|
||||
return os.system(f'docker push {image_tag2}')
|
||||
|
||||
|
||||
class AscendSwiftImageBuilder(SwiftImageBuilder):
|
||||
|
||||
def init_args(self, args) -> Any:
|
||||
if not args.base_image:
|
||||
# other vision search for: https://hub.docker.com/r/ascendai/cann/tags
|
||||
args.base_image = 'swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:8.3.rc1-a3-ubuntu22.04-py3.11'
|
||||
return super().init_args(args)
|
||||
|
||||
def generate_dockerfile(self) -> str:
|
||||
extra_content = """
|
||||
RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
|
||||
"""
|
||||
with open('docker/Dockerfile.ascend', 'r') as f:
|
||||
content = f.read()
|
||||
content = content.replace('{base_image}', self.args.base_image)
|
||||
content = content.replace('{extra_content}', extra_content)
|
||||
content = content.replace('{cur_time}', formatted_time)
|
||||
content = content.replace('{install_ms_deps}', 'False')
|
||||
content = content.replace('{modelscope_branch}',
|
||||
self.args.modelscope_branch)
|
||||
content = content.replace('{swift_branch}', self.args.swift_branch)
|
||||
return content
|
||||
|
||||
def image(self) -> str:
|
||||
return (
|
||||
f'{docker_registry}:{self.args.base_image.split(":")[-1]}-torch2.7.1'
|
||||
f'-{self.args.modelscope_version}-ascend-swift-test')
|
||||
|
||||
def push(self):
|
||||
return 0
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--base_image', type=str, default=None)
|
||||
parser.add_argument('--image_type', type=str)
|
||||
@@ -450,6 +482,8 @@ elif args.image_type.lower() == 'llm':
|
||||
builder_cls = LLMImageBuilder
|
||||
elif args.image_type.lower() == 'swift':
|
||||
builder_cls = SwiftImageBuilder
|
||||
elif args.image_type.lower() == 'ascend_swift':
|
||||
builder_cls = AscendSwiftImageBuilder
|
||||
else:
|
||||
raise ValueError(f'Unsupported image_type: {args.image_type}')
|
||||
|
||||
|
||||
@@ -131,7 +131,7 @@ else:
|
||||
if module is not None:
|
||||
module = _patch_pretrained_class([module], wrap=True)
|
||||
else:
|
||||
raise ImportError(
|
||||
raise AttributeError(
|
||||
f'Cannot import available module of {name} in modelscope,'
|
||||
f' or related packages({hf_pkgs})')
|
||||
return module[0]
|
||||
|
||||
@@ -129,6 +129,11 @@ class CreateCMD(CLICommand):
|
||||
help=
|
||||
'Source of the AIGC model. `USER_UPLOAD`, `TRAINED_FROM_MODELSCOPE` or `TRAINED_FROM_ALIYUN_FC`.'
|
||||
)
|
||||
aigc_group.add_argument(
|
||||
'--base_model_sub_type',
|
||||
type=str,
|
||||
default='',
|
||||
help='Base model sub type, e.g., Qwen_Edit_2509')
|
||||
|
||||
parser.set_defaults(func=subparser_func)
|
||||
|
||||
@@ -191,6 +196,7 @@ class CreateCMD(CLICommand):
|
||||
base_model_id=self.args.base_model_id,
|
||||
path_in_repo=self.args.path_in_repo,
|
||||
model_source=self.args.model_source,
|
||||
base_model_sub_type=self.args.base_model_sub_type,
|
||||
)
|
||||
|
||||
# Convert visibility string to int for the API call
|
||||
|
||||
@@ -19,7 +19,8 @@ from http import HTTPStatus
|
||||
from http.cookiejar import CookieJar
|
||||
from os.path import expanduser
|
||||
from pathlib import Path
|
||||
from typing import Any, BinaryIO, Dict, Iterable, List, Optional, Tuple, Union
|
||||
from typing import (Any, BinaryIO, Dict, Iterable, List, Literal, Optional,
|
||||
Tuple, Union)
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import json
|
||||
@@ -304,6 +305,7 @@ class HubApi:
|
||||
'ModelPath': aigc_model.model_path,
|
||||
'TriggerWords': aigc_model.trigger_words,
|
||||
'ModelSource': aigc_model.model_source,
|
||||
'SubVisionFoundation': aigc_model.base_model_sub_type,
|
||||
})
|
||||
|
||||
if aigc_model.official_tags:
|
||||
@@ -394,7 +396,9 @@ class HubApi:
|
||||
'WeightsName': aigc_model.weight_filename,
|
||||
'WeightsSha256': aigc_model.weight_sha256,
|
||||
'WeightsSize': aigc_model.weight_size,
|
||||
'TriggerWords': aigc_model.trigger_words
|
||||
'TriggerWords': aigc_model.trigger_words,
|
||||
'AigcType': aigc_model.aigc_type,
|
||||
'VisionFoundation': aigc_model.base_model_type
|
||||
}
|
||||
|
||||
else:
|
||||
@@ -455,7 +459,7 @@ class HubApi:
|
||||
model_id: str,
|
||||
revision: Optional[str] = DEFAULT_MODEL_REVISION,
|
||||
endpoint: Optional[str] = None
|
||||
) -> str:
|
||||
) -> dict:
|
||||
"""Get model information at ModelScope
|
||||
|
||||
Args:
|
||||
@@ -2304,13 +2308,21 @@ class HubApi:
|
||||
allow_patterns = allow_patterns if allow_patterns else None
|
||||
ignore_patterns = ignore_patterns if ignore_patterns else None
|
||||
|
||||
# Ignore .git folder
|
||||
# Ignore .git .cache folders
|
||||
if ignore_patterns is None:
|
||||
ignore_patterns = []
|
||||
elif isinstance(ignore_patterns, str):
|
||||
ignore_patterns = [ignore_patterns]
|
||||
ignore_patterns += DEFAULT_IGNORE_PATTERNS
|
||||
|
||||
# Cover the ignore patterns if both allow and ignore patterns are provided
|
||||
if allow_patterns is not None:
|
||||
if '**' in allow_patterns:
|
||||
ignore_patterns = []
|
||||
ignore_patterns = [
|
||||
p for p in ignore_patterns if p not in allow_patterns
|
||||
]
|
||||
|
||||
commit_message = (
|
||||
commit_message if commit_message is not None else f'Upload to {repo_id} on ModelScope hub'
|
||||
)
|
||||
@@ -2867,6 +2879,91 @@ class HubApi:
|
||||
'total_files': len(to_delete)
|
||||
}
|
||||
|
||||
def set_repo_visibility(self,
|
||||
repo_id: str,
|
||||
repo_type: Literal['model', 'dataset'],
|
||||
visibility: Literal['private', 'public'],
|
||||
token: Union[str, None] = None
|
||||
) -> dict:
|
||||
"""
|
||||
Set the visibility of a repo.
|
||||
|
||||
Args:
|
||||
repo_id (str): The repo id in the format of `owner_name/repo_name`.
|
||||
repo_type (Literal['model', 'dataset']): The repo type, `model` or `dataset`.
|
||||
visibility (Literal['private', 'public']): The visibility to set, `private` or `public`.
|
||||
token (Union[str, None]): The access token. If None, will use the cookies from the local cache.
|
||||
See `https://modelscope.cn/my/myaccesstoken` to get your token.
|
||||
|
||||
Returns:
|
||||
dict: The response from the server.
|
||||
"""
|
||||
if not repo_id:
|
||||
raise ValueError('The arg `repo_id` cannot be empty!')
|
||||
|
||||
if visibility not in ['private', 'public']:
|
||||
raise ValueError(f'Invalid visibility: {visibility}, supported visibilities: `private`, `public`')
|
||||
|
||||
visibility_map: Dict[str, int] = {v: k for k, v in VisibilityMap.items()}
|
||||
visibility_code: int = visibility_map.get(visibility, 5)
|
||||
cookies = self.get_cookies(access_token=token, cookies_required=True)
|
||||
|
||||
if repo_type == REPO_TYPE_MODEL:
|
||||
model_info = self.get_model(model_id=repo_id)
|
||||
path = f'{self.endpoint}/api/v1/models/{repo_id}'
|
||||
tasks = model_info.get('Tasks')
|
||||
model_tasks = ''
|
||||
if isinstance(tasks, list) and tasks:
|
||||
first = tasks[0]
|
||||
if isinstance(first, dict) and first:
|
||||
model_tasks = first.get('name')
|
||||
payload = {
|
||||
'ChineseName': model_info.get('ChineseName', ''),
|
||||
'ModelFramework': model_info.get('ModelFramework', 'Pytorch'),
|
||||
'Visibility': visibility_code,
|
||||
'ProtectedMode': 2,
|
||||
'ApprovalMode': model_info.get('ApprovalMode', 2),
|
||||
'Description': model_info.get('Description', ''),
|
||||
'AigcType': model_info.get('AigcType', ''),
|
||||
'VisionFoundation': model_info.get('VisionFoundation', ''),
|
||||
'ModelCover': model_info.get('ModelCover', ''),
|
||||
'SubScientificField': model_info.get('SubScientificField', None),
|
||||
'ScientificField': model_info.get('NEXA', {}).get('ScientificField', ''),
|
||||
'Source': model_info.get('NEXA', {}).get('Source', ''),
|
||||
'ModelTask': model_tasks,
|
||||
'License': model_info.get('License', ''),
|
||||
}
|
||||
elif repo_type == REPO_TYPE_DATASET:
|
||||
|
||||
repo_id_parts = repo_id.split('/')
|
||||
if len(repo_id_parts) != 2 or not all(repo_id_parts):
|
||||
raise ValueError(f'Invalid dataset repo_id: {repo_id}, should be in format of `owner/dataset_name`')
|
||||
|
||||
dataset_idx, _ = self.get_dataset_id_and_type(
|
||||
dataset_name=repo_id_parts[1],
|
||||
namespace=repo_id_parts[0],
|
||||
)
|
||||
|
||||
path = f'{self.endpoint}/api/v1/datasets/{dataset_idx}'
|
||||
payload = {
|
||||
'Visibility': visibility_code,
|
||||
'ProtectedMode': 2,
|
||||
}
|
||||
else:
|
||||
raise ValueError(f'Invalid repo type: {repo_type}, supported repos: {REPO_TYPE_SUPPORT}')
|
||||
|
||||
r = self.session.put(
|
||||
path,
|
||||
json=payload,
|
||||
cookies=cookies,
|
||||
headers=self.builder_headers(self.headers))
|
||||
|
||||
raise_for_http_status(r)
|
||||
resp = r.json()
|
||||
raise_on_error(resp)
|
||||
|
||||
return resp
|
||||
|
||||
|
||||
class ModelScopeConfig:
|
||||
path_credential = expanduser(MODELSCOPE_CREDENTIALS_PATH)
|
||||
|
||||
@@ -120,9 +120,9 @@ def handle_http_response(response: requests.Response,
|
||||
http_error_msg = 'The request model: %s does not exist!' % (model_id)
|
||||
elif 403 == response.status_code:
|
||||
if cookies is None:
|
||||
http_error_msg = f'Authentication token does not exist, \
|
||||
failed to access model {model_id} which may not exist \
|
||||
or may be private. Please login first.'
|
||||
http_error_msg = (
|
||||
f'Authentication token does not exist, failed to access model {model_id} '
|
||||
'which may not exist or may be private. Please login first.')
|
||||
|
||||
else:
|
||||
http_error_msg = f'The authentication token is invalid, failed to access model {model_id}.'
|
||||
|
||||
@@ -19,6 +19,7 @@ from modelscope.utils.logger import get_logger
|
||||
from modelscope.utils.thread_utils import thread_executor
|
||||
from .api import HubApi, ModelScopeConfig
|
||||
from .callback import ProgressCallback
|
||||
from .constants import DEFAULT_MAX_WORKERS
|
||||
from .errors import InvalidParameter
|
||||
from .file_download import (create_temporary_directory_and_cache,
|
||||
download_file, get_file_download_url)
|
||||
@@ -42,7 +43,7 @@ def snapshot_download(
|
||||
local_dir: Optional[str] = None,
|
||||
allow_patterns: Optional[Union[List[str], str]] = None,
|
||||
ignore_patterns: Optional[Union[List[str], str]] = None,
|
||||
max_workers: int = 8,
|
||||
max_workers: Optional[int] = None,
|
||||
repo_id: str = None,
|
||||
repo_type: Optional[str] = REPO_TYPE_MODEL,
|
||||
enable_file_lock: Optional[bool] = None,
|
||||
@@ -112,6 +113,8 @@ def snapshot_download(
|
||||
f'Invalid repo type: {repo_type}, only support: {REPO_TYPE_SUPPORT}'
|
||||
)
|
||||
|
||||
max_workers = max_workers or DEFAULT_MAX_WORKERS
|
||||
|
||||
if revision is None:
|
||||
revision = DEFAULT_DATASET_REVISION if repo_type == REPO_TYPE_DATASET else DEFAULT_MODEL_REVISION
|
||||
|
||||
|
||||
@@ -82,6 +82,7 @@ class AigcModel:
|
||||
trigger_words: Optional[List[str]] = None,
|
||||
official_tags: Optional[List[str]] = None,
|
||||
model_source: Optional[str] = 'USER_UPLOAD',
|
||||
base_model_sub_type: Optional[str] = '',
|
||||
):
|
||||
"""
|
||||
Initializes the AigcModel helper.
|
||||
@@ -99,6 +100,7 @@ class AigcModel:
|
||||
official_tags (List[str], optional): Official tags for the AIGC model. Defaults to None.
|
||||
model_source (str, optional): Source of the model.
|
||||
`USER_UPLOAD`, `TRAINED_FROM_MODELSCOPE` or `TRAINED_FROM_ALIYUN_FC`. Defaults to 'USER_UPLOAD'.
|
||||
base_model_sub_type (str, Optional): Sub vision foundation model type. Defaults to ''. e.g. `SD_1_5`
|
||||
"""
|
||||
self.model_path = model_path
|
||||
self.aigc_type = aigc_type
|
||||
@@ -106,6 +108,7 @@ class AigcModel:
|
||||
self.tag = tag
|
||||
self.description = description
|
||||
self.model_source = model_source
|
||||
self.base_model_sub_type = base_model_sub_type
|
||||
# Process cover images - convert local paths to base64 data URLs
|
||||
if cover_images is not None:
|
||||
processed_cover_images = []
|
||||
@@ -391,6 +394,7 @@ class AigcModel:
|
||||
'trigger_words': self.trigger_words,
|
||||
'official_tags': self.official_tags,
|
||||
'model_source': self.model_source,
|
||||
'base_model_sub_type': self.base_model_sub_type,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -436,6 +436,25 @@ def _patch_pretrained_class(all_imported_modules, wrap=False):
|
||||
**ignore_file_pattern_kwargs))
|
||||
|
||||
all_available_modules.append(var)
|
||||
|
||||
def get_class_from_dynamic_module(class_reference, *args, **kwargs):
|
||||
from transformers.dynamic_module_utils import origin_get_class_from_dynamic_module
|
||||
if '--' in class_reference:
|
||||
repo_id, class_reference = class_reference.split('--')
|
||||
if not os.path.exists(repo_id):
|
||||
from modelscope import snapshot_download
|
||||
repo_id = snapshot_download(repo_id)
|
||||
class_reference = repo_id + '--' + class_reference
|
||||
return origin_get_class_from_dynamic_module(class_reference, *args,
|
||||
**kwargs)
|
||||
|
||||
from transformers import dynamic_module_utils
|
||||
if not hasattr(dynamic_module_utils,
|
||||
'origin_get_class_from_dynamic_module'):
|
||||
dynamic_module_utils.origin_get_class_from_dynamic_module = dynamic_module_utils.get_class_from_dynamic_module
|
||||
dynamic_module_utils.get_class_from_dynamic_module = get_class_from_dynamic_module
|
||||
from transformers.models.auto import configuration_auto
|
||||
configuration_auto.get_class_from_dynamic_module = get_class_from_dynamic_module
|
||||
return all_available_modules
|
||||
|
||||
|
||||
@@ -469,6 +488,13 @@ def _unpatch_pretrained_class(all_imported_modules):
|
||||
except: # noqa
|
||||
pass
|
||||
|
||||
from transformers import dynamic_module_utils
|
||||
if hasattr(dynamic_module_utils, 'origin_get_class_from_dynamic_module'):
|
||||
dynamic_module_utils.get_class_from_dynamic_module = dynamic_module_utils.origin_get_class_from_dynamic_module
|
||||
from transformers.models.auto import configuration_auto
|
||||
configuration_auto.get_class_from_dynamic_module = dynamic_module_utils.origin_get_class_from_dynamic_module
|
||||
delattr(dynamic_module_utils, 'origin_get_class_from_dynamic_module')
|
||||
|
||||
|
||||
def _patch_hub():
|
||||
import huggingface_hub
|
||||
|
||||
@@ -30,8 +30,6 @@ DEFAULT_IGNORE_PATTERNS = [
|
||||
'*/.cache',
|
||||
'**/.cache/**',
|
||||
]
|
||||
# Forbidden to commit these folders
|
||||
FORBIDDEN_FOLDERS = ['.git', '.cache']
|
||||
|
||||
UploadMode = Literal['lfs', 'normal']
|
||||
|
||||
@@ -555,11 +553,7 @@ def _validate_path_in_repo(path_in_repo: str) -> str:
|
||||
f"Invalid `path_in_repo` in CommitOperation: '{path_in_repo}'")
|
||||
if path_in_repo.startswith('./'):
|
||||
path_in_repo = path_in_repo[2:]
|
||||
for forbidden in FORBIDDEN_FOLDERS:
|
||||
if any(part == forbidden for part in path_in_repo.split('/')):
|
||||
raise ValueError(
|
||||
f"Invalid `path_in_repo` in CommitOperation: cannot update files under a '{forbidden}/' folder (path:"
|
||||
f" '{path_in_repo}').")
|
||||
|
||||
return path_in_repo
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Make sure to modify __release_datetime__ to release time when making official release.
|
||||
__version__ = '1.32.0'
|
||||
__version__ = '1.33.0'
|
||||
# default release datetime for branches under active development is set
|
||||
# to be a time far-far-away-into-the-future
|
||||
__release_datetime__ = '2025-11-07 12:00:00'
|
||||
__release_datetime__ = '2025-12-10 23:59:59'
|
||||
|
||||
@@ -108,6 +108,9 @@ class HubRevisionTest(unittest.TestCase):
|
||||
cache_dir=temp_cache_dir)
|
||||
assert os.path.exists(file_path)
|
||||
|
||||
@unittest.skip(
|
||||
'snapshot_download cannot exercise release-mode auto-selection while'
|
||||
' DEFAULT_MODEL_REVISION forces a concrete branch (master).')
|
||||
def test_snapshot_download_revision(self):
|
||||
with mock.patch.dict(os.environ, self.modified_environ, clear=True):
|
||||
self.prepare_repo_data_and_tag()
|
||||
@@ -205,6 +208,7 @@ class HubRevisionTest(unittest.TestCase):
|
||||
model_file_download(
|
||||
self.model_id,
|
||||
download_model_file_name2,
|
||||
revision=None,
|
||||
cache_dir=temp_cache_dir)
|
||||
version.__release_datetime__ = t2
|
||||
logger.info('Setting __release_datetime__ to: %s' % t2)
|
||||
|
||||
Reference in New Issue
Block a user