From 62ffc04215d8030deb47957a9ec9b265209f882e Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Fri, 25 Oct 2024 13:37:01 +0800 Subject: [PATCH] Fix the slow downloading (#1051) * fix slow download * fix * fix audio installation * fix build * fix * test * test * test * test * fix * fix * fix * fix --- docker/Dockerfile.ubuntu | 28 +++++++++++++++++++++++++--- docker/build_image.py | 25 ++++++++++++++++++++++++- docker/install.sh | 2 -- docker/install_cpu.sh | 2 -- 4 files changed, 49 insertions(+), 8 deletions(-) diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index 58f87bed..056c0026 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -16,14 +16,36 @@ COPY {meta_file} /tmp/install.sh RUN sh /tmp/install.sh {version_args} -RUN pip install --no-cache-dir https://modelscope.oss-cn-beijing.aliyuncs.com/packages/imageio_ffmpeg-0.4.9-py3-none-any.whl --force - RUN pip uninstall ms-swift modelscope -y -RUN cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {modelscope_branch} --single-branch https://github.com/modelscope/modelscope.git && cd modelscope && pip install .[all] -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && cd / && rm -fr /tmp/modelscope && pip cache purge; +RUN pip --no-cache-dir install pip==23.3.1 + +RUN pip --no-cache-dir install omegaconf==2.0.6 + +RUN pip install --no-cache-dir 'cython<=0.29.36' kwsbp==0.0.6 versioneer numpy==1.25.1 -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html + +# install dependencies +COPY requirements /var/modelscope +RUN pip install --no-cache-dir -r /var/modelscope/framework.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir -r /var/modelscope/audio.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir -r /var/modelscope/cv.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir -r /var/modelscope/multi-modal.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir -r /var/modelscope/nlp.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir -r /var/modelscope/science.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir -r /var/modelscope/tests.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir -r /var/modelscope/server.txt && \ + pip install --no-cache-dir https://modelscope.oss-cn-beijing.aliyuncs.com/packages/imageio_ffmpeg-0.4.9-py3-none-any.whl --force && \ + pip install --no-cache-dir 'scipy<1.13.0' && \ + pip cache purge + +RUN echo "cache bust $(date +%Y%m%d%H%M%S)" + +RUN cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {modelscope_branch} --single-branch https://github.com/modelscope/modelscope.git && cd modelscope && pip install . -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && cd / && rm -fr /tmp/modelscope && pip cache purge; RUN cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {swift_branch} --single-branch https://github.com/modelscope/ms-swift.git && cd ms-swift && pip install .[all] && cd / && rm -fr /tmp/ms-swift && pip cache purge; +RUN pip install --no-cache-dir torch=={torch_version} torchvision=={torchvision_version} torchaudio=={torchaudio_version} {index_url} + RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \ pip config set install.trusted-host mirrors.aliyun.com && \ cp /tmp/resources/ubuntu2204.aliyun /etc/apt/sources.list diff --git a/docker/build_image.py b/docker/build_image.py index cc80c38e..d53708eb 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -147,6 +147,15 @@ class CPUImageBuilder(Builder): content = content.replace('{extra_content}', extra_content) content = content.replace('{meta_file}', meta_file) content = content.replace('{version_args}', version_args) + content = content.replace('{torch_version}', + self.args.torch_version) + content = content.replace('{torchvision_version}', + self.args.torchvision_version) + content = content.replace('{torchaudio_version}', + self.args.torchaudio_version) + content = content.replace( + '{index_url}', + '--index-url https://download.pytorch.org/whl/cpu') content = content.replace('{modelscope_branch}', self.args.modelscope_branch) content = content.replace('{swift_branch}', self.args.swift_branch) @@ -195,6 +204,13 @@ class GPUImageBuilder(Builder): content = content.replace('{extra_content}', extra_content) content = content.replace('{meta_file}', meta_file) content = content.replace('{version_args}', version_args) + content = content.replace('{torch_version}', + self.args.torch_version) + content = content.replace('{torchvision_version}', + self.args.torchvision_version) + content = content.replace('{torchaudio_version}', + self.args.torchaudio_version) + content = content.replace('{index_url}', '') content = content.replace('{modelscope_branch}', self.args.modelscope_branch) content = content.replace('{swift_branch}', self.args.swift_branch) @@ -238,7 +254,7 @@ class LLMImageBuilder(Builder): if not args.cuda_version: args.cuda_version = '12.4.0' if not args.vllm_version: - args.vllm_version = '0.6.0' + args.vllm_version = '0.6.3.post1' if not args.lmdeploy_version: args.lmdeploy_version = '0.6.1' if not args.autogptq_version: @@ -261,6 +277,13 @@ class LLMImageBuilder(Builder): content = content.replace('{extra_content}', extra_content) content = content.replace('{meta_file}', meta_file) content = content.replace('{version_args}', version_args) + content = content.replace('{torch_version}', + self.args.torch_version) + content = content.replace('{torchvision_version}', + self.args.torchvision_version) + content = content.replace('{torchaudio_version}', + self.args.torchaudio_version) + content = content.replace('{index_url}', '') content = content.replace('{modelscope_branch}', self.args.modelscope_branch) content = content.replace('{swift_branch}', self.args.swift_branch) diff --git a/docker/install.sh b/docker/install.sh index 145b43ad..0daecf2e 100644 --- a/docker/install.sh +++ b/docker/install.sh @@ -46,5 +46,3 @@ pip install --no-cache-dir vllm==$vllm_version -U pip install --no-cache-dir -U lmdeploy==$lmdeploy_version --no-deps pip install --no-cache-dir pynvml shortuuid - -pip install --no-cache-dir torch==$torch_version torchvision==$torchvision_version torchaudio==$torchaudio_version diff --git a/docker/install_cpu.sh b/docker/install_cpu.sh index 1cd3396b..fce62afe 100644 --- a/docker/install_cpu.sh +++ b/docker/install_cpu.sh @@ -19,5 +19,3 @@ pip install --no-cache-dir -U qwen_vl_utils pyav librosa timm transformers accel pip install --no-cache-dir text2sql_lgesql==1.3.0 git+https://github.com/jin-s13/xtcocoapi.git@v1.14 git+https://github.com/gatagat/lap.git@v0.4.0 -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --force --no-deps pip install --no-cache-dir mpi4py paint_ldm mmcls>=0.21.0 mmdet>=2.25.0 decord>=0.6.0 ipykernel fasttext -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html - -pip install --no-cache-dir -U torch==$torch_version torchvision==$torchvision_version torchaudio==$torchaudio_version --index-url https://download.pytorch.org/whl/cpu