diff --git a/.dev_scripts/dockerci.sh b/.dev_scripts/dockerci.sh index 4f66073c..d892f61c 100644 --- a/.dev_scripts/dockerci.sh +++ b/.dev_scripts/dockerci.sh @@ -43,7 +43,6 @@ do -e HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT \ -e TEST_ACCESS_TOKEN_CITEST=$TEST_ACCESS_TOKEN_CITEST \ -e TEST_ACCESS_TOKEN_SDKDEV=$TEST_ACCESS_TOKEN_SDKDEV \ - -e TEST_LEVEL=$TEST_LEVEL \ -e MODELSCOPE_ENVIRONMENT='ci' \ -e TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN \ -e MODEL_TAG_URL=$MODEL_TAG_URL \ @@ -66,7 +65,6 @@ do -e HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT \ -e TEST_ACCESS_TOKEN_CITEST=$TEST_ACCESS_TOKEN_CITEST \ -e TEST_ACCESS_TOKEN_SDKDEV=$TEST_ACCESS_TOKEN_SDKDEV \ - -e TEST_LEVEL=$TEST_LEVEL \ -e MODELSCOPE_ENVIRONMENT='ci' \ -e TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN \ -e MODEL_TAG_URL=$MODEL_TAG_URL \ diff --git a/README.md b/README.md index 0656b117..eb65c053 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -itest` +


diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index 103d7141..fd51af0e 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -20,11 +20,10 @@ ARG INSTALL_MS_DEPS={install_ms_deps} COPY requirements /var/modelscope RUN pip uninstall ms-swift modelscope -y && pip --no-cache-dir install pip==23.* -U && \ - pip install --no-cache-dir apex -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ if [ "$INSTALL_MS_DEPS" = "True" ]; then \ pip --no-cache-dir install omegaconf==2.0.6 && \ + pip install 'editdistance==0.8.1' && \ pip install --no-cache-dir 'cython<=0.29.36' versioneer 'numpy<2.0' -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ - pip install --no-cache-dir kwsbp==0.0.6 -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ pip install --no-cache-dir -r /var/modelscope/framework.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ pip install --no-cache-dir -r /var/modelscope/audio.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ pip install --no-cache-dir -r /var/modelscope/cv.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ @@ -39,6 +38,13 @@ if [ "$INSTALL_MS_DEPS" = "True" ]; then \ pip install --no-cache-dir funtextprocessing typeguard==2.13.3 scikit-learn -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ pip install --no-cache-dir text2sql_lgesql==1.3.0 git+https://github.com/jin-s13/xtcocoapi.git@v1.14 git+https://github.com/gatagat/lap.git@v0.4.0 -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --force --no-deps && \ pip install --no-cache-dir mmcls>=0.21.0 mmdet>=2.25.0 decord>=0.6.0 mpi4py paint_ldm ipykernel fasttext -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip uninstall ddpm_guided_diffusion -y && \ + pip install --no-cache-dir 'blobfile>=1.0.5' && \ + pip install 'ddpm_guided_diffusion' -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --no-index && \ + pip uninstall shotdetect_scenedetect_lgss -y && \ + pip install 'shotdetect_scenedetect_lgss' -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --no-index && \ + pip uninstall MinDAEC -y && \ + pip install https://modelscope.oss-cn-beijing.aliyuncs.com/releases/dependencies/MinDAEC-0.0.2-py3-none-any.whl && \ pip cache purge; \ else \ pip install --no-cache-dir -r /var/modelscope/framework.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ @@ -49,6 +55,7 @@ ARG CUR_TIME={cur_time} RUN echo $CUR_TIME RUN sh /tmp/install.sh {version_args} && \ + pip install --no-cache-dir xformers==0.0.27 && \ curl -fsSL https://ollama.com/install.sh | sh && \ pip install --no-cache-dir -U funasr scikit-learn && \ pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils pyav librosa timm transformers accelerate peft trl safetensors && \ diff --git a/docker/Dockerfile.ubuntu_base b/docker/Dockerfile.ubuntu_base index 903c9930..a80f008c 100644 --- a/docker/Dockerfile.ubuntu_base +++ b/docker/Dockerfile.ubuntu_base @@ -186,7 +186,15 @@ RUN if [ "$USE_GPU" = "True" ] ; then \ RUN if [ "$USE_GPU" = "True" ] ; then \ export TORCH_CUDA_ARCH_LIST="6.0;6.1;7.0;7.5;8.0;8.9;9.0;8.6+PTX" && pip install --no-cache-dir git+https://github.com/gxd1994/Pointnet2.PyTorch.git@master#subdirectory=pointnet2; \ else \ - echo 'cpu unsupport Pointnet2'; \ + echo 'CPU env does not support Pointnet2'; \ + fi + +# NVIDIA apex building +RUN if [ "$USE_GPU" = "True" ] ; then \ + pip install --no-cache-dir packaging &&\ + bash /tmp/install_apex.sh; \ + else \ + echo 'CPU env does not support NVIDIA Apex.'; \ fi @@ -203,13 +211,6 @@ ARG TENSORFLOW_VERSION={tf_version} cd /tmp && git clone -b ms_build --single-branch https://github.com/tastelikefeet/mmcv.git && cd mmcv && MMCV_WITH_OPS=1 MAX_JOBS=32 pip install . && cd / && rm -fr /tmp/mmcv && pip cache purge; \ fi - # This limits the cuda121 version - RUN if [ "$USE_GPU" = "True" ] ; then \ - pip install --no-cache-dir --force tinycudann==1.7 -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html; \ - else \ - echo 'cpu not install tinycudann'; \ - fi - - RUN pip install --no-cache-dir fairseq + RUN pip install --no-cache-dir https://github.com/liyaodev/fairseq/releases/download/v0.12.3.1/fairseq-0.12.3.1-cp311-cp311-linux_x86_64.whl # for py311 ENTRYPOINT [] diff --git a/docker/build_image.py b/docker/build_image.py index 4bfe9db2..268ba658 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -31,7 +31,7 @@ class Builder: if not args.cuda_version: args.cuda_version = '12.1.0' if not args.vllm_version: - args.vllm_version = '0.5.3' + args.vllm_version = '0.7.2' if not args.lmdeploy_version: args.lmdeploy_version = '0.6.2' if not args.autogptq_version: diff --git a/docker/scripts/install_unifold.sh b/docker/scripts/install_unifold.sh index 0e9f3682..ba7a1870 100644 --- a/docker/scripts/install_unifold.sh +++ b/docker/scripts/install_unifold.sh @@ -8,5 +8,4 @@ apt-get update && apt-get install -y hmmer kalign curl cmake \ && ln -s /opt/hhsuite/bin/* /usr/bin \ && popd \ && rm -rf /tmp/hh-suite \ - && pip install --no-cache-dir unicore -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html \ && pip install --no-cache-dir biopython ipdb diff --git a/modelscope/models/cv/nerf_recon_4k/network/utils.py b/modelscope/models/cv/nerf_recon_4k/network/utils.py index aa8c3d66..6983d36a 100644 --- a/modelscope/models/cv/nerf_recon_4k/network/utils.py +++ b/modelscope/models/cv/nerf_recon_4k/network/utils.py @@ -5,7 +5,6 @@ from collections import defaultdict import mcubes import numpy as np -import tinycudann as tcnn import torch import torch.nn as nn import torch.nn.functional as F @@ -166,6 +165,8 @@ def normalize(dat, inp_scale, tgt_scale): def cleanup(): + import tinycudann as tcnn + gc.collect() torch.cuda.empty_cache() tcnn.free_temporary_memory() diff --git a/modelscope/models/cv/nerf_recon_acc/network/nerf.py b/modelscope/models/cv/nerf_recon_acc/network/nerf.py index 972d2d85..64ffae14 100644 --- a/modelscope/models/cv/nerf_recon_acc/network/nerf.py +++ b/modelscope/models/cv/nerf_recon_acc/network/nerf.py @@ -1,7 +1,6 @@ # The implementation is modified from nerfacc, made publicly available under the MIT License # at https://github.com/KAIR-BAIR/nerfacc/blob/master/examples/radiance_fields/ngp.py import numpy as np -import tinycudann as tcnn import torch import torch.nn as nn from nerfacc import ContractionType, OccupancyGrid, ray_marching, rendering @@ -113,6 +112,8 @@ trunc_exp = _TruncExp.apply class VolumeDensity(nn.Module): def __init__(self, config): + import tinycudann as tcnn + super().__init__() self.config = config self.radius = self.config.radius diff --git a/modelscope/models/cv/nerf_recon_acc/network/utils.py b/modelscope/models/cv/nerf_recon_acc/network/utils.py index aa8c3d66..6983d36a 100644 --- a/modelscope/models/cv/nerf_recon_acc/network/utils.py +++ b/modelscope/models/cv/nerf_recon_acc/network/utils.py @@ -5,7 +5,6 @@ from collections import defaultdict import mcubes import numpy as np -import tinycudann as tcnn import torch import torch.nn as nn import torch.nn.functional as F @@ -166,6 +165,8 @@ def normalize(dat, inp_scale, tgt_scale): def cleanup(): + import tinycudann as tcnn + gc.collect() torch.cuda.empty_cache() tcnn.free_temporary_memory() diff --git a/modelscope/pipelines/nlp/text_generation_pipeline.py b/modelscope/pipelines/nlp/text_generation_pipeline.py index 8d3a5a07..2f0c4b7c 100644 --- a/modelscope/pipelines/nlp/text_generation_pipeline.py +++ b/modelscope/pipelines/nlp/text_generation_pipeline.py @@ -440,7 +440,7 @@ class QWenTextGenerationPipeline(Pipeline): class SeqGPTPipeline(Pipeline): def __init__(self, model: Union[Model, str], **kwargs): - from modelscope.utils.hf_util import AutoTokenizer + from modelscope import AutoTokenizer if isinstance(model, str): model_dir = snapshot_download( diff --git a/requirements/audio/audio_codec.txt b/requirements/audio/audio_codec.txt index c7ac8b2b..bb6097b9 100644 --- a/requirements/audio/audio_codec.txt +++ b/requirements/audio/audio_codec.txt @@ -1 +1 @@ -funcodec>=0.2.0 +ms-funcodec>=0.2.0 diff --git a/requirements/audio/audio_tts.txt b/requirements/audio/audio_tts.txt index 5cff1b28..972bb992 100644 --- a/requirements/audio/audio_tts.txt +++ b/requirements/audio/audio_tts.txt @@ -15,7 +15,7 @@ protobuf ptflops ptyprocess>=0.7.0 pygments>=2.12.0 -pysptk>=0.1.15,<0.1.19 +#pysptk>=0.1.15,<0.1.19 pytorch_wavelets PyWavelets>=1.0.0 scikit-learn @@ -23,6 +23,6 @@ sox tensorboardx tqdm traitlets>=5.3.0 -ttsfrd>=0.1.2 +#ttsfrd>=0.1.2 # not supported for py311 or above unidecode wcwidth>=0.2.5 diff --git a/requirements/multi-modal.txt b/requirements/multi-modal.txt index 6c974b56..b2f767c5 100644 --- a/requirements/multi-modal.txt +++ b/requirements/multi-modal.txt @@ -2,9 +2,11 @@ accelerate cloudpickle decord>=0.6.0 diffusers>=0.25.0 -# 0.12.1 has issue of No such file or directory: 'fairseq/version.txt' -fairseq==0.12.2 ftfy>=6.0.3 +# 0.12.1 has issue of No such file or directory: 'fairseq/version.txt' +# 0.12.2 not support py311 +#fairseq==0.12.2 +https://github.com/liyaodev/fairseq/releases/download/v0.12.3.1/fairseq-0.12.3.1-cp311-cp311-linux_x86_64.whl librosa==0.10.1 opencv-python pycocoevalcap>=1.2