Update build file and workflow (#1551)

* update build file * update setup file * update fairseq dep * fix error log
2025-12-14 15:27:42 +01:00 · 2025-12-02 10:50:39 +08:00
parent 6f3200dfc2
commit 9cb7ce7029
9 changed files with 39 additions and 29 deletions
--- a/.github/workflows/daily_regression.yaml
+++ b/.github/workflows/daily_regression.yaml
@@ -32,7 +32,6 @@ jobs:

      - name: Fetch LFS objects
        run: |
-          git lfs install --local --force
          git lfs pull

      - name: Run unittest
--- a/.github/workflows/publish.yaml
+++ b/.github/workflows/publish.yaml
@@ -24,8 +24,7 @@ jobs:
      - name: Build ModelScope
        # Build AST template before packaging
        run: |
-          python -c "from modelscope.utils.ast_utils import generate_ast_template; generate_ast_template()"
-          python setup.py sdist bdist_wheel
+          make whl
      - name: Publish package to PyPI
        run: |
          pip install twine
--- a/3
+++ b/3
@@ -18,7 +18,8 @@ test:

 .PHONY: whl
 whl:
-	python setup.py sdist bdist_wheel
+	python -c "from modelscope.utils.ast_utils import generate_ast_template; generate_ast_template()"
+	python setup.py sdist --dist-dir $(WHL_BUILD_DIR)/dist bdist_wheel --dist-dir $(WHL_BUILD_DIR)/dist

 .PHONY: clean
 clean:
--- a/docker/Dockerfile.ubuntu
+++ b/docker/Dockerfile.ubuntu
@@ -16,7 +16,7 @@ COPY {meta_file} /tmp/install.sh

 ARG INSTALL_MS_DEPS={install_ms_deps}

-ARG INSTALL_MEGATRON_DEPS={install_megatron_deps}
+ARG IMAGE_TYPE={image_type}

 # install dependencies
 COPY requirements /var/modelscope
@@ -56,7 +56,7 @@ fi
 ARG CUR_TIME={cur_time}
 RUN echo $CUR_TIME

-RUN sh /tmp/install.sh {version_args} && \
+RUN bash /tmp/install.sh {version_args} && \
    curl -fsSL https://ollama.com/install.sh | sh && \
    pip install --no-cache-dir -U funasr scikit-learn && \
    pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa timm transformers accelerate peft trl safetensors && \
@@ -75,10 +75,7 @@ RUN sh /tmp/install.sh {version_args} && \
    cp /tmp/resources/ubuntu2204.aliyun /etc/apt/sources.list


-RUN if [ "$INSTALL_MS_DEPS" = "True" ]; then \
-    pip install --no-cache-dir huggingface-hub transformers peft -U; \
-fi; \
-if [ "$INSTALL_MEGATRON_DEPS" = "True" ]; then \
+RUN if [ "$IMAGE_TYPE" = "swift" ]; then \
    pip install "sglang[all]<0.5" "math_verify==0.5.2" "gradio<5.33" -U && \
    pip install liger_kernel wandb swanlab nvitop pre-commit "transformers<4.57" "trl<0.21" huggingface-hub -U && \
    SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])") && echo $SITE_PACKAGES && \
@@ -88,6 +85,11 @@ if [ "$INSTALL_MEGATRON_DEPS" = "True" ]; then \
    cd apex && git checkout e13873debc4699d39c6861074b9a3b2a02327f92 && pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" ./ && \
    cd / && rm -fr /tmp/apex && pip cache purge; \
    pip install git+https://github.com/NVIDIA/Megatron-LM.git@core_r0.13.0; \
+elif [ "$IMAGE_TYPE" = "llm" ]; then \
+    pip install --no-cache-dir huggingface-hub transformers peft diffusers -U; \
+    pip uninstall autoawq -y; \
+else \
+    pip install "transformers<4.56" "tokenizers<0.22" "trl<0.23" "diffusers<0.35" --no-dependencies; \
 fi

 # install nvm and set node version to 18
@@ -97,7 +99,6 @@ RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh | b
    nvm install 22 && \
    nvm use 22

-ENV SETUPTOOLS_USE_DISTUTILS=stdlib
 ENV VLLM_USE_MODELSCOPE=True
 ENV LMDEPLOY_USE_MODELSCOPE=True
 ENV MODELSCOPE_CACHE=/mnt/workspace/.cache/modelscope/hub
--- a/docker/build_image.py
+++ b/docker/build_image.py
@@ -162,7 +162,7 @@ class CPUImageBuilder(Builder):
            content = content.replace('{version_args}', version_args)
            content = content.replace('{cur_time}', formatted_time)
            content = content.replace('{install_ms_deps}', 'True')
-            content = content.replace('{install_megatron_deps}', 'False')
+            content = content.replace('{image_type}', 'cpu')
            content = content.replace('{torch_version}',
                                      self.args.torch_version)
            content = content.replace('{torchvision_version}',
@@ -207,6 +207,8 @@ class GPUImageBuilder(Builder):
        # pushd ~ popd is to solve the tf cannot use gpu problem.
        extra_content = """
 RUN pip install tf-keras==2.16.0 --no-dependencies && \
+    pip install onnx==1.18.0 --no-dependencies && \
+    pip install deepspeed==0.17.4 --no-dependencies && \
    pip install --no-cache-dir torchsde jupyterlab torchmetrics==0.11.4 basicsr pynvml shortuuid && \
    CUDA_HOME=/usr/local/cuda TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0 7.5 8.0 8.6 8.9 9.0" \
        pip install --no-cache-dir  'git+https://github.com/facebookresearch/detectron2.git'
@@ -230,7 +232,7 @@ RUN pushd $(dirname $(python -c 'print(__import__("tensorflow").__file__)'))  &&
            content = content.replace('{version_args}', version_args)
            content = content.replace('{cur_time}', formatted_time)
            content = content.replace('{install_ms_deps}', 'True')
-            content = content.replace('{install_megatron_deps}', 'False')
+            content = content.replace('{image_type}', 'gpu')
            content = content.replace('{torch_version}',
                                      self.args.torch_version)
            content = content.replace('{torchvision_version}',
@@ -273,19 +275,19 @@ class LLMImageBuilder(Builder):
            # A mirrored image of nvidia/cuda:12.4.0-devel-ubuntu22.04
            args.base_image = 'nvidia/cuda:12.4.0-devel-ubuntu22.04'
        if not args.torch_version:
-            args.torch_version = '2.6.0'
-            args.torchaudio_version = '2.6.0'
-            args.torchvision_version = '0.21.0'
+            args.torch_version = '2.8.0'
+            args.torchaudio_version = '2.8.0'
+            args.torchvision_version = '0.23.0'
        if not args.cuda_version:
            args.cuda_version = '12.4.0'
        if not args.vllm_version:
-            args.vllm_version = '0.8.5.post1'
+            args.vllm_version = '0.11.0'
        if not args.lmdeploy_version:
-            args.lmdeploy_version = '0.9.1'
+            args.lmdeploy_version = '0.10.1'
        if not args.autogptq_version:
            args.autogptq_version = '0.7.1'
        if not args.flashattn_version:
-            args.flashattn_version = '2.7.1.post4'
+            args.flashattn_version = '2.7.4.post1'
        return args

    def generate_dockerfile(self) -> str:
@@ -306,7 +308,7 @@ class LLMImageBuilder(Builder):
            content = content.replace('{version_args}', version_args)
            content = content.replace('{cur_time}', formatted_time)
            content = content.replace('{install_ms_deps}', 'False')
-            content = content.replace('{install_megatron_deps}', 'False')
+            content = content.replace('{image_type}', 'llm')
            content = content.replace('{torch_version}',
                                      self.args.torch_version)
            content = content.replace('{torchvision_version}',
@@ -382,7 +384,7 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
            content = content.replace('{version_args}', version_args)
            content = content.replace('{cur_time}', formatted_time)
            content = content.replace('{install_ms_deps}', 'False')
-            content = content.replace('{install_megatron_deps}', 'True')
+            content = content.replace('{image_type}', 'swift')
            content = content.replace('{torch_version}',
                                      self.args.torch_version)
            content = content.replace('{torchvision_version}',
--- a/docker/install.sh
+++ b/docker/install.sh
@@ -28,6 +28,8 @@ pip install --no-cache-dir triton auto-gptq==$autogptq_version -U && pip cache p
 if [[ "$(printf '%s\n' "0.6.0" "$vllm_version" | sort -V | head -n1)" = "0.6.0" ]]; then
    # vllm_version is >= 0.6.0
    pip install --no-cache-dir vllm==$vllm_version && pip cache purge
+else
+    echo "vllm_version < 0.6.0, skipping installation. (vllm_version = $vllm_version)"
 fi

 # pip uninstall -y torch-scatter && TORCH_CUDA_ARCH_LIST="6.0;6.1;6.2;7.0;7.5;8.0;8.6;8.9;9.0" pip install --no-cache-dir -U torch-scatter
--- a/modelscope/hub/errors.py
+++ b/modelscope/hub/errors.py
@@ -120,11 +120,12 @@ def handle_http_response(response: requests.Response,
        http_error_msg = 'The request model: %s does not exist!' % (model_id)
    elif 403 == response.status_code:
        if cookies is None:
-            http_error_msg = 'Authentication token does not exist, '
-            'failed to access model {model_id} which may not exist or may be '
-            'private. Please login first.'
+            http_error_msg = (
+                f'Authentication token does not exist, failed to access model {model_id} '
+                'which may not exist or may be private. Please login first.')
+
        else:
-            http_error_msg = 'The authentication token is invalid, failed to access model {model_id}.'
+            http_error_msg = f'The authentication token is invalid, failed to access model {model_id}.'
    elif 400 <= response.status_code < 500:
        http_error_msg = u'%s Client Error: %s, Request id: %s for url: %s' % (
            response.status_code, reason, request_id, response.url)
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -46,6 +46,8 @@ hub        = {file = ["requirements/hub.txt"]}
 datasets   = {file = ["requirements/datasets.txt"]}
 framework  = {file = ["requirements/framework.txt"]}
 server     = {file = ["requirements/server.txt"]}
+docs       = {file = ["requirements/docs.txt"]}
+tests      = {file = ["requirements/tests.txt"]}

 # domain specific with framework base
 cv          = {file = ["requirements/framework.txt", "requirements/cv.txt"]}
@@ -59,9 +61,12 @@ audio_codec   = {file = ["requirements/framework.txt", "requirements/audio/audio
 audio_tts     = {file = ["requirements/framework.txt", "requirements/audio/audio_tts.txt"]}
 audio_kws     = {file = ["requirements/framework.txt", "requirements/audio/audio_kws.txt"]}
 audio_signal  = {file = ["requirements/framework.txt", "requirements/audio/audio_signal.txt"]}
-
-docs       = {file = ["requirements/docs.txt"]}
-tests      = {file = ["requirements/tests.txt"]}
+audio         = {file = ["requirements/framework.txt",
+                        "requirements/audio/audio_asr.txt",
+                        "requirements/audio/audio_codec.txt",
+                        "requirements/audio/audio_tts.txt",
+                        "requirements/audio/audio_kws.txt",
+                        "requirements/audio/audio_signal.txt"]}

 # skip audio requirements due to its hard dependency which may cause installation failure
 all = {file = [
--- a/requirements/multi-modal.txt
+++ b/requirements/multi-modal.txt
@@ -5,7 +5,7 @@ diffusers>=0.25.0
 # 0.12.1 has issue of No such file or directory: 'fairseq/version.txt'
 # 0.12.2 not support py311
 #fairseq==0.12.2
-fairseq @ https://github.com/liyaodev/fairseq/releases/download/v0.12.3.1/fairseq-0.12.3.1-cp311-cp311-linux_x86_64.whl ; python_version == "3.11" and platform_system == "Linux" and platform_machine == "x86_64"
+fairseq-fixed==0.12.3.1
 ftfy>=6.0.3
 librosa==0.10.1
 opencv-python