diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu
index f3c326ec..ffb268b2 100644
--- a/docker/Dockerfile.ubuntu
+++ b/docker/Dockerfile.ubuntu
@@ -22,6 +22,12 @@ COPY {meta_file} /tmp/install.sh
 
 RUN sh /tmp/install.sh {version_args}
 
+pip uninstall ms-swift modelscope -y
+
+cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b $modelscope_branch  --single-branch https://github.com/modelscope/modelscope.git && cd modelscope && pip install .[all] -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && cd / && rm -fr /tmp/modelscope && pip cache purge;
+
+cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b $swift_branch  --single-branch https://github.com/modelscope/ms-swift.git && cd ms-swift && pip install .[all] && cd / && rm -fr /tmp/ms-swift && pip cache purge;
+
 ENV SETUPTOOLS_USE_DISTUTILS=stdlib
 ENV VLLM_USE_MODELSCOPE=True
 ENV LMDEPLOY_USE_MODELSCOPE=True
diff --git a/docker/install.sh b/docker/install.sh
index 20ebba83..a76b07de 100644
--- a/docker/install.sh
+++ b/docker/install.sh
@@ -47,10 +47,4 @@ pip install --no-cache-dir vllm==$vllm_version -U
 
 pip install --no-cache-dir -U lmdeploy==$lmdeploy_version --no-deps
 
-pip install --no-cache-dir pynvml shortuuid
-
-pip uninstall ms-swift modelscope -y
-
-cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b $modelscope_branch  --single-branch https://github.com/modelscope/modelscope.git && cd modelscope && pip install .[all] -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && cd / && rm -fr /tmp/modelscope && pip cache purge;
-
-cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b $swift_branch  --single-branch https://github.com/modelscope/ms-swift.git && cd ms-swift && pip install .[all] && cd / && rm -fr /tmp/ms-swift && pip cache purge;
+pip install --no-cache-dir pynvml shortuuid
\ No newline at end of file
diff --git a/docker/install_cpu.sh b/docker/install_cpu.sh
index 7ecfd9a7..edac4734 100644
--- a/docker/install_cpu.sh
+++ b/docker/install_cpu.sh
@@ -20,10 +20,4 @@ pip install --no-cache-dir -U qwen_vl_utils pyav librosa timm transformers accel
 
 pip install --no-cache-dir text2sql_lgesql==1.3.0 git+https://github.com/jin-s13/xtcocoapi.git@v1.14 git+https://github.com/gatagat/lap.git@v0.4.0 -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --force --no-deps
 
-pip install --no-cache-dir mpi4py paint_ldm mmcls>=0.21.0 mmdet>=2.25.0 decord>=0.6.0 ipykernel fasttext -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html
-
-pip uninstall ms-swift modelscope -y
-
-cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b $modelscope_branch  --single-branch https://github.com/modelscope/modelscope.git && cd modelscope && pip install .[all] -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && cd / && rm -fr /tmp/modelscope && pip cache purge;
-
-cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b $swift_branch  --single-branch https://github.com/modelscope/ms-swift.git && cd ms-swift && pip install .[all] && cd / && rm -fr /tmp/ms-swift && pip cache purge;
+pip install --no-cache-dir mpi4py paint_ldm mmcls>=0.21.0 mmdet>=2.25.0 decord>=0.6.0 ipykernel fasttext -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html
\ No newline at end of file