mirror of
https://github.com/modelscope/modelscope.git
synced 2026-09-01 19:49:03 +02:00
Refactor Dockerfile (#1036)
This commit is contained in:
tastelikefeet
@@ -1,168 +0,0 @@
|
||||
#!/bin/bash
|
||||
# default values.
|
||||
BASE_CPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu
|
||||
BASE_GPU_CUDA113_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu:20.04-cuda11.3.0-cudnn8-devel
|
||||
BASE_GPU_CUDA117_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu:20.04-cuda11.7.1-cudnn8-devel
|
||||
BASE_GPU_CUDA118_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu:20.04-cuda11.8.0-cudnn8-devel
|
||||
BASE_GPU_CUDA121_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu:22.04-cuda12.1.0-cudnn8-devel
|
||||
BASE_GPU_CUDA122_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu:22.04-cuda11.2.2-cudnn8-devel
|
||||
MODELSCOPE_REPO_ADDRESS=reg.docker.alibaba-inc.com/modelscope/modelscope
|
||||
python_version=3.7.13
|
||||
torch_version=1.11.0
|
||||
cuda_version=11.7.1
|
||||
cudatoolkit_version=11.3
|
||||
tensorflow_version=1.15.5
|
||||
os_version=20.04
|
||||
version=None
|
||||
is_cpu=False
|
||||
is_dryrun=False
|
||||
function usage(){
|
||||
echo "usage: build.sh "
|
||||
echo " --os=ubuntu_version set ubuntu os version, default: 20.04"
|
||||
echo " --python=python_version set python version, default: $python_version"
|
||||
echo " --cuda=cuda_version set cuda version,only[11.3.0, 11.7.1], fefault: $cuda_version"
|
||||
echo " --torch=torch_version set pytorch version, fefault: $torch_version"
|
||||
echo " --tensorflow=tensorflow_version set tensorflow version, default: $tensorflow_version"
|
||||
echo " --test option for run test before push image, only push on ci test pass"
|
||||
echo " --cpu option for build cpu version"
|
||||
echo " --push option for push image to remote repo"
|
||||
echo " --dryrun create Dockerfile not build"
|
||||
}
|
||||
for i in "$@"; do
|
||||
case $i in
|
||||
--os=*)
|
||||
os_version="${i#*=}"
|
||||
shift
|
||||
;;
|
||||
--python=*)
|
||||
python_version="${i#*=}"
|
||||
shift
|
||||
;;
|
||||
--cuda=*)
|
||||
cuda_version="${i#*=}"
|
||||
shift # pytorch version
|
||||
;;
|
||||
--torch=*)
|
||||
torch_version="${i#*=}"
|
||||
shift # pytorch version
|
||||
;;
|
||||
--tensorflow=*)
|
||||
tensorflow_version="${i#*=}"
|
||||
shift # tensorflow version
|
||||
;;
|
||||
--version=*)
|
||||
version="${i#*=}"
|
||||
shift # version
|
||||
;;
|
||||
--cpu)
|
||||
is_cpu=True
|
||||
shift # is cpu image
|
||||
;;
|
||||
--push)
|
||||
is_push=True
|
||||
shift # option for push image to remote repo
|
||||
;;
|
||||
--dryrun)
|
||||
is_dryrun=True
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
-*|--*)
|
||||
echo "Unknown option $i"
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$cuda_version" == 11.3.0 ]; then
|
||||
echo "Building base image cuda11.3.0"
|
||||
BASE_GPU_IMAGE=$os_version-$cudatoolkit_version-cudnn8-devel
|
||||
cudatoolkit_version=cu113
|
||||
elif [ "$cuda_version" == 11.7.1 ]; then
|
||||
echo "Building base image cuda11.7.1"
|
||||
cudatoolkit_version=cu117
|
||||
BASE_GPU_IMAGE=$BASE_GPU_CUDA117_IMAGE
|
||||
elif [ "$cuda_version" == 11.8.0 ]; then
|
||||
echo "Building base image cuda11.8.0"
|
||||
cudatoolkit_version=cu118
|
||||
BASE_GPU_IMAGE=$MODELSCOPE_REPO_ADDRESS:$os_version-cuda$cuda_version-cudnn8-devel
|
||||
elif [ "$cuda_version" == 12.1.0 ]; then
|
||||
cudatoolkit_version=cu121
|
||||
BASE_GPU_IMAGE=$BASE_GPU_CUDA121_IMAGE
|
||||
else
|
||||
echo "Unsupport cuda version: $cuda_version"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$is_cpu" == "True" ]; then
|
||||
export BASE_IMAGE=$BASE_CPU_IMAGE:$os_version
|
||||
base_tag=ubuntu$os_version
|
||||
export USE_GPU=False
|
||||
else
|
||||
export BASE_IMAGE=$BASE_GPU_IMAGE
|
||||
base_tag=ubuntu$os_version-cuda$cuda_version
|
||||
export USE_GPU=True
|
||||
fi
|
||||
|
||||
if [[ $python_version == 3.7* ]]; then
|
||||
base_tag=$base_tag-py37
|
||||
elif [[ $python_version == 3.8* ]]; then
|
||||
base_tag=$base_tag-py38
|
||||
elif [[ $python_version == 3.10* ]]; then
|
||||
base_tag=$base_tag-py310
|
||||
else
|
||||
echo "Unsupport python version: $python_version"
|
||||
exit 1
|
||||
fi
|
||||
# target_image_tag=$base_tag-torch$torch_version-tf$tensorflow_version-base
|
||||
# cpu no tensorflow
|
||||
if [ "$is_cpu" == "True" ]; then
|
||||
target_image_tag=$base_tag-torch$torch_version-base
|
||||
else
|
||||
target_image_tag=$base_tag-torch$torch_version-tf$tensorflow_version-base
|
||||
fi
|
||||
|
||||
export IMAGE_TO_BUILD=$MODELSCOPE_REPO_ADDRESS:$target_image_tag
|
||||
export PYTHON_VERSION=$python_version
|
||||
export TORCH_VERSION=$torch_version
|
||||
export CUDATOOLKIT_VERSION=$cudatoolkit_version
|
||||
export TENSORFLOW_VERSION=$tensorflow_version
|
||||
echo "From: $BASE_IMAGE build: $target_image_tag"
|
||||
echo -e "Building image with:\npython$python_version\npytorch$torch_version\ntensorflow:$tensorflow_version\ncudatoolkit:$cudatoolkit_version\ncpu:$is_cpu\n"
|
||||
docker_file_content=`cat docker/Dockerfile.ubuntu_base`
|
||||
printf "$docker_file_content" > Dockerfile
|
||||
|
||||
if [ "$is_dryrun" == "True" ]; then
|
||||
echo 'Dockerfile created'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# DOCKER_BUILDKIT=0
|
||||
while true
|
||||
do
|
||||
DOCKER_BUILDKIT=0 docker build -t $IMAGE_TO_BUILD \
|
||||
--build-arg USE_GPU \
|
||||
--build-arg BASE_IMAGE \
|
||||
--build-arg PYTHON_VERSION \
|
||||
--build-arg TORCH_VERSION \
|
||||
--build-arg CUDATOOLKIT_VERSION \
|
||||
--build-arg TENSORFLOW_VERSION \
|
||||
-f Dockerfile .
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "Image build done"
|
||||
break
|
||||
else
|
||||
echo "Running docker build command error, we will retry"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$is_push" == "True" ]; then
|
||||
echo "Pushing image: $IMAGE_TO_BUILD"
|
||||
docker push $IMAGE_TO_BUILD
|
||||
fi
|
||||
@@ -1,204 +0,0 @@
|
||||
#!/bin/bash
|
||||
# default values.
|
||||
#BASE_PY37_CPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-py37-torch1.11.0-tf1.15.5-base
|
||||
#BASE_PY38_CPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-py38-torch1.11.0-tf1.15.5-base
|
||||
#BASE_PY38_GPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-cuda11.3.0-py38-torch1.11.0-tf1.15.5-base
|
||||
#BASE_PY38_GPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-cuda11.7.1-py38-torch2.0.1-tf1.15.5-base
|
||||
#BASE_PY38_GPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-cuda11.7.1-py38-torch1.13.1-tf2.6.0-base
|
||||
#BASE_PY37_GPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-cuda11.3.0-py37-torch1.11.0-tf1.15.5-base
|
||||
MODELSCOPE_REPO_ADDRESS=reg.docker.alibaba-inc.com/modelscope/modelscope
|
||||
python_version=3.7.13
|
||||
torch_version=1.11.0
|
||||
cudatoolkit_version=11.7
|
||||
tensorflow_version=1.15.5
|
||||
modelscope_version=None
|
||||
cuda_version=11.7.1
|
||||
is_dsw=False
|
||||
is_cpu=False
|
||||
build_branch='master'
|
||||
function usage(){
|
||||
echo "usage: build.sh "
|
||||
echo " --python=python_version set python version, default: $python_version"
|
||||
echo " --cuda=cuda_version set cuda version,only[11.3.0, 11.7.1], fefault: $cuda_version"
|
||||
echo " --torch=torch_version set pytorch version, fefault: $torch_version"
|
||||
echo " --tensorflow=tensorflow_version set tensorflow version, default: $tensorflow_version"
|
||||
echo " --modelscope=modelscope_version set modelscope version, default: $modelscope_version"
|
||||
echo " --branch=build_branch set modelscope build branch, default: $build_branch"
|
||||
echo " --cpu option for build cpu version"
|
||||
echo " --dsw option for build dsw version"
|
||||
echo " --push option for push image to remote repo"
|
||||
}
|
||||
for i in "$@"; do
|
||||
case $i in
|
||||
--python=*)
|
||||
python_version="${i#*=}"
|
||||
shift
|
||||
;;
|
||||
--cuda=*)
|
||||
cuda_version="${i#*=}"
|
||||
if [ "$cuda_version" == "11.3.0" ]; then
|
||||
cudatoolkit_version=11.3
|
||||
elif [ "$cuda_version" == "11.7.1" ]; then
|
||||
cudatoolkit_version=11.7
|
||||
elif [ "$cuda_version" == "11.8.0" ]; then
|
||||
cudatoolkit_version=11.8
|
||||
elif [ "$cuda_version" == "12.1.0" ]; then
|
||||
cudatoolkit_version=12.1
|
||||
else
|
||||
echo "Unsupport cuda version $cuda_version"
|
||||
exit 1
|
||||
fi
|
||||
shift # pytorch version
|
||||
;;
|
||||
--torch=*)
|
||||
torch_version="${i#*=}"
|
||||
shift # pytorch version
|
||||
;;
|
||||
--tensorflow=*)
|
||||
tensorflow_version="${i#*=}"
|
||||
shift # tensorflow version
|
||||
;;
|
||||
--cudatoolkit=*)
|
||||
cudatoolkit_version="${i#*=}"
|
||||
shift # cudatoolkit for pytorch
|
||||
;;
|
||||
--modelscope=*)
|
||||
modelscope_version="${i#*=}"
|
||||
shift # modelscope version
|
||||
;;
|
||||
--branch=*)
|
||||
build_branch="${i#*=}"
|
||||
shift # build branch
|
||||
;;
|
||||
--cpu)
|
||||
is_cpu=True
|
||||
shift # is cpu image
|
||||
;;
|
||||
--dsw)
|
||||
is_dsw=True
|
||||
shift # is dsw, will set dsw cache location
|
||||
;;
|
||||
--push)
|
||||
is_push=True
|
||||
shift # option for push image to remote repo
|
||||
;;
|
||||
--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
-*|--*)
|
||||
echo "Unknown option $i"
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$modelscope_version" == "None" ]; then
|
||||
echo "ModelScope version must specify!"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$is_cpu" == "True" ]; then
|
||||
base_tag=ubuntu20.04
|
||||
export USE_GPU=False
|
||||
else
|
||||
base_tag=ubuntu20.04-cuda$cuda_version
|
||||
export USE_GPU=True
|
||||
fi
|
||||
|
||||
if [[ $python_version == 3.7* ]]; then
|
||||
if [ "$is_cpu" == "True" ]; then
|
||||
echo "Building python3.7 cpu image"
|
||||
export BASE_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-py37-torch$torch_version-tf$tensorflow_version-base
|
||||
else
|
||||
echo "Building python3.7 gpu image"
|
||||
export BASE_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-cuda$cuda_version-py37-torch$torch_version-tf$tensorflow_version-base
|
||||
fi
|
||||
base_tag=$base_tag-py37
|
||||
elif [[ $python_version == 3.8* ]]; then
|
||||
if [ "$is_cpu" == "True" ]; then
|
||||
echo "Building python3.8 cpu image"
|
||||
export BASE_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-py38-torch$torch_version-tf$tensorflow_version-base
|
||||
else
|
||||
echo "Building python3.8 gpu image"
|
||||
export BASE_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-cuda$cuda_version-py38-torch$torch_version-tf$tensorflow_version-base
|
||||
fi
|
||||
base_tag=$base_tag-py38
|
||||
elif [[ $python_version == 3.10* ]]; then
|
||||
if [ "$is_cpu" == "True" ]; then
|
||||
echo "Building python3.10 cpu image"
|
||||
base_tag=ubuntu22.04-py310
|
||||
export BASE_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu22.04-py310-torch$torch_version-base
|
||||
else
|
||||
echo "Building python3.10 gpu image"
|
||||
base_tag=ubuntu22.04-cuda$cuda_version-py310
|
||||
# reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu22.04-cuda12.1.0-py310-torch2.1.0-tf2.14.0-base
|
||||
export BASE_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu22.04-cuda$cuda_version-py310-torch$torch_version-tf$tensorflow_version-base
|
||||
fi
|
||||
else
|
||||
echo "Unsupport python version: $python_version"
|
||||
exit 1
|
||||
fi
|
||||
# cpu not intall tensorflow
|
||||
# target_image_tag=$base_tag-torch$torch_version-tf$tensorflow_version-$modelscope_version-test
|
||||
if [ "$is_cpu" == "True" ]; then
|
||||
target_image_tag=$base_tag-torch$torch_version-$modelscope_version-test
|
||||
else
|
||||
target_image_tag=$base_tag-torch$torch_version-tf$tensorflow_version-$modelscope_version-test
|
||||
fi
|
||||
export IMAGE_TO_BUILD=$MODELSCOPE_REPO_ADDRESS:$target_image_tag
|
||||
export PYTHON_VERSION=$python_version
|
||||
export TORCH_VERSION=$torch_version
|
||||
export CUDATOOLKIT_VERSION=$cudatoolkit_version
|
||||
export TENSORFLOW_VERSION=$tensorflow_version
|
||||
echo -e "Building image with:\npython$python_version\npytorch$torch_version\ntensorflow:$tensorflow_version\ncudatoolkit:$cudatoolkit_version\ncpu:$is_cpu\nis_ci:$is_ci_test\nis_dsw:$is_dsw\n"
|
||||
echo -e "Base iamge: $BASE_IMAGE"
|
||||
docker_file_content=`cat docker/Dockerfile.ubuntu`
|
||||
|
||||
BUILD_HASH_ID=$(git rev-parse HEAD)
|
||||
# install thrid part library
|
||||
docker_file_content="${docker_file_content} \nRUN export COMMIT_ID=$BUILD_HASH_ID && pip install --no-cache-dir -U adaseq pai-easycv && pip install --no-cache-dir -U 'ms-swift' 'decord' 'qwen_vl_utils' 'pyav' 'librosa' 'funasr' autoawq 'timm>0.9.5' 'transformers' 'accelerate' 'peft' 'optimum' 'trl' 'outlines<0.1'"
|
||||
|
||||
docker_file_content="${docker_file_content} \nRUN pip uninstall modelscope -y && export COMMIT_ID=$BUILD_HASH_ID && cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b $build_branch --single-branch $REPO_URL && cd modelscope && pip install . && cd / && rm -fr /tmp/modelscope && pip cache purge;"
|
||||
|
||||
echo "$is_dsw"
|
||||
if [ "$is_dsw" == "False" ]; then
|
||||
echo "Not DSW image"
|
||||
else
|
||||
echo "Building dsw image will need set ModelScope lib cache location."
|
||||
docker_file_content="${docker_file_content} \nENV MODELSCOPE_CACHE=/mnt/workspace/.cache/modelscope"
|
||||
# pre compile extension
|
||||
docker_file_content="${docker_file_content} \nRUN pip uninstall -y tb-nightly tensorboard && pip install --no-cache-dir -U tensorboard && TORCH_CUDA_ARCH_LIST='6.0 6.1 7.0 7.5 8.0 8.9 9.0 8.6+PTX' python -c 'from modelscope.utils.pre_compile import pre_compile_all;pre_compile_all()'"
|
||||
fi
|
||||
|
||||
|
||||
docker_file_content="${docker_file_content} \n RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
|
||||
pip config set install.trusted-host mirrors.aliyun.com && \
|
||||
cp /tmp/resources/ubuntu2204.aliyun /etc/apt/sources.list "
|
||||
|
||||
printf "$docker_file_content" > Dockerfile
|
||||
|
||||
while true
|
||||
do
|
||||
docker build --progress=plain -t $IMAGE_TO_BUILD \
|
||||
--build-arg USE_GPU \
|
||||
--build-arg BASE_IMAGE \
|
||||
--build-arg PYTHON_VERSION \
|
||||
--build-arg TORCH_VERSION \
|
||||
--build-arg CUDATOOLKIT_VERSION \
|
||||
--build-arg TENSORFLOW_VERSION \
|
||||
-f Dockerfile .
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "Image build done"
|
||||
break
|
||||
else
|
||||
echo "Running docker build command error, we will retry"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$is_push" == "True" ]; then
|
||||
echo "Pushing image: $IMAGE_TO_BUILD"
|
||||
docker push $IMAGE_TO_BUILD
|
||||
fi
|
||||
@@ -1,7 +0,0 @@
|
||||
#sudo docker run --name zwm_maas -v /home/wenmeng.zwm/workspace:/home/wenmeng.zwm/workspace --net host -ti reg.docker.alibaba-inc.com/pai-dlc/tensorflow-training:2.3-gpu-py36-cu101-ubuntu18.04 bash
|
||||
#sudo docker run --name zwm_maas_pytorch -v /home/wenmeng.zwm/workspace:/home/wenmeng.zwm/workspace --net host -ti reg.docker.alibaba-inc.com/pai-dlc/pytorch-training:1.10PAI-gpu-py36-cu113-ubuntu18.04 bash
|
||||
CONTAINER_NAME=modelscope-dev
|
||||
IMAGE_NAME=registry.cn-shanghai.aliyuncs.com/modelscope/modelscope
|
||||
IMAGE_VERSION=v0.1.1-16-g62856fa-devel
|
||||
MOUNT_DIR=/home/wenmeng.zwm/workspace
|
||||
sudo docker run --name $CONTAINER_NAME -v $MOUNT_DIR:$MOUNT_DIR --net host -ti ${IMAGE_NAME}:${IMAGE_VERSION} bash
|
||||
Reference in New Issue
Block a user