From 9cb7ce7029ed8deddf863507eafbc3850d2cafc0 Mon Sep 17 00:00:00 2001 From: Yunlin Mao Date: Tue, 2 Dec 2025 10:50:39 +0800 Subject: [PATCH] Update build file and workflow (#1551) * update build file * update setup file * update fairseq dep * fix error log --- .github/workflows/daily_regression.yaml | 1 - .github/workflows/publish.yaml | 3 +-- Makefile | 3 ++- docker/Dockerfile.ubuntu | 15 ++++++++------- docker/build_image.py | 22 ++++++++++++---------- docker/install.sh | 2 ++ modelscope/hub/errors.py | 9 +++++---- pyproject.toml | 11 ++++++++--- requirements/multi-modal.txt | 2 +- 9 files changed, 39 insertions(+), 29 deletions(-) diff --git a/.github/workflows/daily_regression.yaml b/.github/workflows/daily_regression.yaml index 73d971e6..bf73d3ea 100644 --- a/.github/workflows/daily_regression.yaml +++ b/.github/workflows/daily_regression.yaml @@ -32,7 +32,6 @@ jobs: - name: Fetch LFS objects run: | - git lfs install --local --force git lfs pull - name: Run unittest diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 0eadd339..49972e1f 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -24,8 +24,7 @@ jobs: - name: Build ModelScope # Build AST template before packaging run: | - python -c "from modelscope.utils.ast_utils import generate_ast_template; generate_ast_template()" - python setup.py sdist bdist_wheel + make whl - name: Publish package to PyPI run: | pip install twine diff --git a/Makefile b/Makefile index 96532199..92385886 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,8 @@ test: .PHONY: whl whl: - python setup.py sdist bdist_wheel + python -c "from modelscope.utils.ast_utils import generate_ast_template; generate_ast_template()" + python setup.py sdist --dist-dir $(WHL_BUILD_DIR)/dist bdist_wheel --dist-dir $(WHL_BUILD_DIR)/dist .PHONY: clean clean: diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index e889c8d7..ec5e70a1 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -16,7 +16,7 @@ COPY {meta_file} /tmp/install.sh ARG INSTALL_MS_DEPS={install_ms_deps} -ARG INSTALL_MEGATRON_DEPS={install_megatron_deps} +ARG IMAGE_TYPE={image_type} # install dependencies COPY requirements /var/modelscope @@ -56,7 +56,7 @@ fi ARG CUR_TIME={cur_time} RUN echo $CUR_TIME -RUN sh /tmp/install.sh {version_args} && \ +RUN bash /tmp/install.sh {version_args} && \ curl -fsSL https://ollama.com/install.sh | sh && \ pip install --no-cache-dir -U funasr scikit-learn && \ pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa timm transformers accelerate peft trl safetensors && \ @@ -75,10 +75,7 @@ RUN sh /tmp/install.sh {version_args} && \ cp /tmp/resources/ubuntu2204.aliyun /etc/apt/sources.list -RUN if [ "$INSTALL_MS_DEPS" = "True" ]; then \ - pip install --no-cache-dir huggingface-hub transformers peft -U; \ -fi; \ -if [ "$INSTALL_MEGATRON_DEPS" = "True" ]; then \ +RUN if [ "$IMAGE_TYPE" = "swift" ]; then \ pip install "sglang[all]<0.5" "math_verify==0.5.2" "gradio<5.33" -U && \ pip install liger_kernel wandb swanlab nvitop pre-commit "transformers<4.57" "trl<0.21" huggingface-hub -U && \ SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])") && echo $SITE_PACKAGES && \ @@ -88,6 +85,11 @@ if [ "$INSTALL_MEGATRON_DEPS" = "True" ]; then \ cd apex && git checkout e13873debc4699d39c6861074b9a3b2a02327f92 && pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" ./ && \ cd / && rm -fr /tmp/apex && pip cache purge; \ pip install git+https://github.com/NVIDIA/Megatron-LM.git@core_r0.13.0; \ +elif [ "$IMAGE_TYPE" = "llm" ]; then \ + pip install --no-cache-dir huggingface-hub transformers peft diffusers -U; \ + pip uninstall autoawq -y; \ +else \ + pip install "transformers<4.56" "tokenizers<0.22" "trl<0.23" "diffusers<0.35" --no-dependencies; \ fi # install nvm and set node version to 18 @@ -97,7 +99,6 @@ RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh | b nvm install 22 && \ nvm use 22 -ENV SETUPTOOLS_USE_DISTUTILS=stdlib ENV VLLM_USE_MODELSCOPE=True ENV LMDEPLOY_USE_MODELSCOPE=True ENV MODELSCOPE_CACHE=/mnt/workspace/.cache/modelscope/hub diff --git a/docker/build_image.py b/docker/build_image.py index 916bb5c2..4cc4075a 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -162,7 +162,7 @@ class CPUImageBuilder(Builder): content = content.replace('{version_args}', version_args) content = content.replace('{cur_time}', formatted_time) content = content.replace('{install_ms_deps}', 'True') - content = content.replace('{install_megatron_deps}', 'False') + content = content.replace('{image_type}', 'cpu') content = content.replace('{torch_version}', self.args.torch_version) content = content.replace('{torchvision_version}', @@ -207,6 +207,8 @@ class GPUImageBuilder(Builder): # pushd ~ popd is to solve the tf cannot use gpu problem. extra_content = """ RUN pip install tf-keras==2.16.0 --no-dependencies && \ + pip install onnx==1.18.0 --no-dependencies && \ + pip install deepspeed==0.17.4 --no-dependencies && \ pip install --no-cache-dir torchsde jupyterlab torchmetrics==0.11.4 basicsr pynvml shortuuid && \ CUDA_HOME=/usr/local/cuda TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0 7.5 8.0 8.6 8.9 9.0" \ pip install --no-cache-dir 'git+https://github.com/facebookresearch/detectron2.git' @@ -230,7 +232,7 @@ RUN pushd $(dirname $(python -c 'print(__import__("tensorflow").__file__)')) && content = content.replace('{version_args}', version_args) content = content.replace('{cur_time}', formatted_time) content = content.replace('{install_ms_deps}', 'True') - content = content.replace('{install_megatron_deps}', 'False') + content = content.replace('{image_type}', 'gpu') content = content.replace('{torch_version}', self.args.torch_version) content = content.replace('{torchvision_version}', @@ -273,19 +275,19 @@ class LLMImageBuilder(Builder): # A mirrored image of nvidia/cuda:12.4.0-devel-ubuntu22.04 args.base_image = 'nvidia/cuda:12.4.0-devel-ubuntu22.04' if not args.torch_version: - args.torch_version = '2.6.0' - args.torchaudio_version = '2.6.0' - args.torchvision_version = '0.21.0' + args.torch_version = '2.8.0' + args.torchaudio_version = '2.8.0' + args.torchvision_version = '0.23.0' if not args.cuda_version: args.cuda_version = '12.4.0' if not args.vllm_version: - args.vllm_version = '0.8.5.post1' + args.vllm_version = '0.11.0' if not args.lmdeploy_version: - args.lmdeploy_version = '0.9.1' + args.lmdeploy_version = '0.10.1' if not args.autogptq_version: args.autogptq_version = '0.7.1' if not args.flashattn_version: - args.flashattn_version = '2.7.1.post4' + args.flashattn_version = '2.7.4.post1' return args def generate_dockerfile(self) -> str: @@ -306,7 +308,7 @@ class LLMImageBuilder(Builder): content = content.replace('{version_args}', version_args) content = content.replace('{cur_time}', formatted_time) content = content.replace('{install_ms_deps}', 'False') - content = content.replace('{install_megatron_deps}', 'False') + content = content.replace('{image_type}', 'llm') content = content.replace('{torch_version}', self.args.torch_version) content = content.replace('{torchvision_version}', @@ -382,7 +384,7 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy content = content.replace('{version_args}', version_args) content = content.replace('{cur_time}', formatted_time) content = content.replace('{install_ms_deps}', 'False') - content = content.replace('{install_megatron_deps}', 'True') + content = content.replace('{image_type}', 'swift') content = content.replace('{torch_version}', self.args.torch_version) content = content.replace('{torchvision_version}', diff --git a/docker/install.sh b/docker/install.sh index b15dab31..661dcc81 100644 --- a/docker/install.sh +++ b/docker/install.sh @@ -28,6 +28,8 @@ pip install --no-cache-dir triton auto-gptq==$autogptq_version -U && pip cache p if [[ "$(printf '%s\n' "0.6.0" "$vllm_version" | sort -V | head -n1)" = "0.6.0" ]]; then # vllm_version is >= 0.6.0 pip install --no-cache-dir vllm==$vllm_version && pip cache purge +else + echo "vllm_version < 0.6.0, skipping installation. (vllm_version = $vllm_version)" fi # pip uninstall -y torch-scatter && TORCH_CUDA_ARCH_LIST="6.0;6.1;6.2;7.0;7.5;8.0;8.6;8.9;9.0" pip install --no-cache-dir -U torch-scatter diff --git a/modelscope/hub/errors.py b/modelscope/hub/errors.py index 2f63edd9..e5fb3621 100644 --- a/modelscope/hub/errors.py +++ b/modelscope/hub/errors.py @@ -120,11 +120,12 @@ def handle_http_response(response: requests.Response, http_error_msg = 'The request model: %s does not exist!' % (model_id) elif 403 == response.status_code: if cookies is None: - http_error_msg = 'Authentication token does not exist, ' - 'failed to access model {model_id} which may not exist or may be ' - 'private. Please login first.' + http_error_msg = ( + f'Authentication token does not exist, failed to access model {model_id} ' + 'which may not exist or may be private. Please login first.') + else: - http_error_msg = 'The authentication token is invalid, failed to access model {model_id}.' + http_error_msg = f'The authentication token is invalid, failed to access model {model_id}.' elif 400 <= response.status_code < 500: http_error_msg = u'%s Client Error: %s, Request id: %s for url: %s' % ( response.status_code, reason, request_id, response.url) diff --git a/pyproject.toml b/pyproject.toml index d3036994..3b95faac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,8 @@ hub = {file = ["requirements/hub.txt"]} datasets = {file = ["requirements/datasets.txt"]} framework = {file = ["requirements/framework.txt"]} server = {file = ["requirements/server.txt"]} +docs = {file = ["requirements/docs.txt"]} +tests = {file = ["requirements/tests.txt"]} # domain specific with framework base cv = {file = ["requirements/framework.txt", "requirements/cv.txt"]} @@ -59,9 +61,12 @@ audio_codec = {file = ["requirements/framework.txt", "requirements/audio/audio audio_tts = {file = ["requirements/framework.txt", "requirements/audio/audio_tts.txt"]} audio_kws = {file = ["requirements/framework.txt", "requirements/audio/audio_kws.txt"]} audio_signal = {file = ["requirements/framework.txt", "requirements/audio/audio_signal.txt"]} - -docs = {file = ["requirements/docs.txt"]} -tests = {file = ["requirements/tests.txt"]} +audio = {file = ["requirements/framework.txt", + "requirements/audio/audio_asr.txt", + "requirements/audio/audio_codec.txt", + "requirements/audio/audio_tts.txt", + "requirements/audio/audio_kws.txt", + "requirements/audio/audio_signal.txt"]} # skip audio requirements due to its hard dependency which may cause installation failure all = {file = [ diff --git a/requirements/multi-modal.txt b/requirements/multi-modal.txt index af12af06..46ac0054 100644 --- a/requirements/multi-modal.txt +++ b/requirements/multi-modal.txt @@ -5,7 +5,7 @@ diffusers>=0.25.0 # 0.12.1 has issue of No such file or directory: 'fairseq/version.txt' # 0.12.2 not support py311 #fairseq==0.12.2 -fairseq @ https://github.com/liyaodev/fairseq/releases/download/v0.12.3.1/fairseq-0.12.3.1-cp311-cp311-linux_x86_64.whl ; python_version == "3.11" and platform_system == "Linux" and platform_machine == "x86_64" +fairseq-fixed==0.12.3.1 ftfy>=6.0.3 librosa==0.10.1 opencv-python