Merge release1.24 to master (#1298)

2025-12-16 16:27:45 +01:00 · 2025-04-07 13:32:09 +08:00
parent 3bfff5cb53
commit 32aefd7428
13 changed files with 35 additions and 25 deletions
--- a/.dev_scripts/dockerci.sh
+++ b/.dev_scripts/dockerci.sh
@@ -43,7 +43,6 @@ do
              -e HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT \
              -e TEST_ACCESS_TOKEN_CITEST=$TEST_ACCESS_TOKEN_CITEST \
              -e TEST_ACCESS_TOKEN_SDKDEV=$TEST_ACCESS_TOKEN_SDKDEV \
-              -e TEST_LEVEL=$TEST_LEVEL \
              -e MODELSCOPE_ENVIRONMENT='ci' \
              -e TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN \
              -e MODEL_TAG_URL=$MODEL_TAG_URL \
@@ -66,7 +65,6 @@ do
              -e HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT \
              -e TEST_ACCESS_TOKEN_CITEST=$TEST_ACCESS_TOKEN_CITEST \
              -e TEST_ACCESS_TOKEN_SDKDEV=$TEST_ACCESS_TOKEN_SDKDEV \
-              -e TEST_LEVEL=$TEST_LEVEL \
              -e MODELSCOPE_ENVIRONMENT='ci' \
              -e TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN \
              -e MODEL_TAG_URL=$MODEL_TAG_URL \
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-itest`
+
 <p align="center">
    <br>
    <img src="https://modelscope.oss-cn-beijing.aliyuncs.com/modelscope.gif" width="400"/>
--- a/docker/Dockerfile.ubuntu
+++ b/docker/Dockerfile.ubuntu
@@ -20,11 +20,10 @@ ARG INSTALL_MS_DEPS={install_ms_deps}
 COPY requirements /var/modelscope

 RUN pip uninstall ms-swift modelscope -y && pip --no-cache-dir install pip==23.* -U && \
-    pip install --no-cache-dir apex -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
 if [ "$INSTALL_MS_DEPS" = "True" ]; then \
    pip --no-cache-dir install omegaconf==2.0.6 && \
+    pip install 'editdistance==0.8.1' && \
    pip install --no-cache-dir 'cython<=0.29.36' versioneer 'numpy<2.0' -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
-    pip install --no-cache-dir kwsbp==0.0.6 -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
    pip install --no-cache-dir -r /var/modelscope/framework.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
    pip install --no-cache-dir -r /var/modelscope/audio.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
    pip install --no-cache-dir -r /var/modelscope/cv.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
@@ -39,6 +38,13 @@ if [ "$INSTALL_MS_DEPS" = "True" ]; then \
    pip install --no-cache-dir funtextprocessing typeguard==2.13.3 scikit-learn -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
    pip install --no-cache-dir text2sql_lgesql==1.3.0 git+https://github.com/jin-s13/xtcocoapi.git@v1.14 git+https://github.com/gatagat/lap.git@v0.4.0 -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --force --no-deps && \
    pip install --no-cache-dir mmcls>=0.21.0 mmdet>=2.25.0 decord>=0.6.0 mpi4py paint_ldm ipykernel fasttext -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
+    pip uninstall ddpm_guided_diffusion -y && \
+    pip install --no-cache-dir 'blobfile>=1.0.5' && \
+    pip install 'ddpm_guided_diffusion' -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --no-index && \
+    pip uninstall shotdetect_scenedetect_lgss -y && \
+    pip install 'shotdetect_scenedetect_lgss' -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --no-index && \
+    pip uninstall MinDAEC -y && \
+    pip install https://modelscope.oss-cn-beijing.aliyuncs.com/releases/dependencies/MinDAEC-0.0.2-py3-none-any.whl && \
    pip cache purge; \
 else \
    pip install --no-cache-dir -r /var/modelscope/framework.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
@@ -49,6 +55,7 @@ ARG CUR_TIME={cur_time}
 RUN echo $CUR_TIME

 RUN sh /tmp/install.sh {version_args} && \
+    pip install --no-cache-dir xformers==0.0.27 && \
    curl -fsSL https://ollama.com/install.sh | sh && \
    pip install --no-cache-dir -U funasr scikit-learn && \
    pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils pyav librosa timm transformers accelerate peft trl safetensors && \
--- a/docker/Dockerfile.ubuntu_base
+++ b/docker/Dockerfile.ubuntu_base
@@ -186,7 +186,15 @@ RUN if [ "$USE_GPU" = "True" ] ; then \
 RUN if [ "$USE_GPU" = "True" ] ; then \
       export TORCH_CUDA_ARCH_LIST="6.0;6.1;7.0;7.5;8.0;8.9;9.0;8.6+PTX" && pip install --no-cache-dir git+https://github.com/gxd1994/Pointnet2.PyTorch.git@master#subdirectory=pointnet2; \
    else \
-     echo 'cpu unsupport Pointnet2'; \
+     echo 'CPU env does not support Pointnet2'; \
+    fi
+
+# NVIDIA apex building
+RUN if [ "$USE_GPU" = "True" ] ; then \
+        pip install --no-cache-dir packaging &&\
+        bash /tmp/install_apex.sh; \
+    else \
+     echo 'CPU env does not support NVIDIA Apex.'; \
    fi


@@ -203,13 +211,6 @@ ARG TENSORFLOW_VERSION={tf_version}
            cd /tmp && git clone -b ms_build --single-branch https://github.com/tastelikefeet/mmcv.git && cd mmcv && MMCV_WITH_OPS=1 MAX_JOBS=32 pip install . && cd / && rm -fr /tmp/mmcv && pip cache purge; \
    fi

- # This limits the cuda121 version
- RUN if [ "$USE_GPU" = "True" ] ; then \
-            pip install --no-cache-dir --force tinycudann==1.7  -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html; \
-        else \
-            echo 'cpu not install tinycudann'; \
-    fi
-
- RUN pip install --no-cache-dir fairseq
+ RUN pip install --no-cache-dir https://github.com/liyaodev/fairseq/releases/download/v0.12.3.1/fairseq-0.12.3.1-cp311-cp311-linux_x86_64.whl  # for py311

 ENTRYPOINT []
--- a/docker/build_image.py
+++ b/docker/build_image.py
@@ -31,7 +31,7 @@ class Builder:
        if not args.cuda_version:
            args.cuda_version = '12.1.0'
        if not args.vllm_version:
-            args.vllm_version = '0.5.3'
+            args.vllm_version = '0.7.2'
        if not args.lmdeploy_version:
            args.lmdeploy_version = '0.6.2'
        if not args.autogptq_version:
--- a/docker/scripts/install_unifold.sh
+++ b/docker/scripts/install_unifold.sh
@@ -8,5 +8,4 @@ apt-get update && apt-get install -y  hmmer kalign curl cmake \
        && ln -s /opt/hhsuite/bin/* /usr/bin \
        && popd \
        && rm -rf /tmp/hh-suite \
-        && pip install --no-cache-dir unicore -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html \
        && pip install --no-cache-dir  biopython ipdb
--- a/modelscope/models/cv/nerf_recon_4k/network/utils.py
+++ b/modelscope/models/cv/nerf_recon_4k/network/utils.py
@@ -5,7 +5,6 @@ from collections import defaultdict

 import mcubes
 import numpy as np
-import tinycudann as tcnn
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
@@ -166,6 +165,8 @@ def normalize(dat, inp_scale, tgt_scale):


 def cleanup():
+    import tinycudann as tcnn
+
    gc.collect()
    torch.cuda.empty_cache()
    tcnn.free_temporary_memory()
--- a/modelscope/models/cv/nerf_recon_acc/network/nerf.py
+++ b/modelscope/models/cv/nerf_recon_acc/network/nerf.py
@@ -1,7 +1,6 @@
 # The implementation is modified from nerfacc, made publicly available under the MIT License
 # at https://github.com/KAIR-BAIR/nerfacc/blob/master/examples/radiance_fields/ngp.py
 import numpy as np
-import tinycudann as tcnn
 import torch
 import torch.nn as nn
 from nerfacc import ContractionType, OccupancyGrid, ray_marching, rendering
@@ -113,6 +112,8 @@ trunc_exp = _TruncExp.apply
 class VolumeDensity(nn.Module):

    def __init__(self, config):
+        import tinycudann as tcnn
+
        super().__init__()
        self.config = config
        self.radius = self.config.radius
--- a/modelscope/models/cv/nerf_recon_acc/network/utils.py
+++ b/modelscope/models/cv/nerf_recon_acc/network/utils.py
@@ -5,7 +5,6 @@ from collections import defaultdict

 import mcubes
 import numpy as np
-import tinycudann as tcnn
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
@@ -166,6 +165,8 @@ def normalize(dat, inp_scale, tgt_scale):


 def cleanup():
+    import tinycudann as tcnn
+
    gc.collect()
    torch.cuda.empty_cache()
    tcnn.free_temporary_memory()
--- a/modelscope/pipelines/nlp/text_generation_pipeline.py
+++ b/modelscope/pipelines/nlp/text_generation_pipeline.py
@@ -440,7 +440,7 @@ class QWenTextGenerationPipeline(Pipeline):
 class SeqGPTPipeline(Pipeline):

    def __init__(self, model: Union[Model, str], **kwargs):
-        from modelscope.utils.hf_util import AutoTokenizer
+        from modelscope import AutoTokenizer

        if isinstance(model, str):
            model_dir = snapshot_download(
--- a/requirements/audio/audio_codec.txt
+++ b/requirements/audio/audio_codec.txt
@@ -1 +1 @@
-funcodec>=0.2.0
+ms-funcodec>=0.2.0
--- a/requirements/audio/audio_tts.txt
+++ b/requirements/audio/audio_tts.txt
@@ -15,7 +15,7 @@ protobuf
 ptflops
 ptyprocess>=0.7.0
 pygments>=2.12.0
-pysptk>=0.1.15,<0.1.19
+#pysptk>=0.1.15,<0.1.19
 pytorch_wavelets
 PyWavelets>=1.0.0
 scikit-learn
@@ -23,6 +23,6 @@ sox
 tensorboardx
 tqdm
 traitlets>=5.3.0
-ttsfrd>=0.1.2
+#ttsfrd>=0.1.2  # not supported for py311 or above
 unidecode
 wcwidth>=0.2.5
--- a/requirements/multi-modal.txt
+++ b/requirements/multi-modal.txt
@@ -2,9 +2,11 @@ accelerate
 cloudpickle
 decord>=0.6.0
 diffusers>=0.25.0
-# 0.12.1 has issue of No such file or directory: 'fairseq/version.txt'
-fairseq==0.12.2
 ftfy>=6.0.3
+# 0.12.1 has issue of No such file or directory: 'fairseq/version.txt'
+# 0.12.2 not support py311
+#fairseq==0.12.2
+https://github.com/liyaodev/fairseq/releases/download/v0.12.3.1/fairseq-0.12.3.1-cp311-cp311-linux_x86_64.whl
 librosa==0.10.1
 opencv-python
 pycocoevalcap>=1.2