docker file py38 and py37 compatible merge

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12722824 * debug * add missing deps * add stanza * fix numpy issue * numpy version * tf case * numpy <= 1.22.0 * fix tf 1.15.5 case * add python38 and python37 compatible docker file * cv add fvcore * add easycv * remove debug code * fix style issue * fix python3.7 python3.8 compatible issue * comment * modify run config * scipy<=1.7.3 * revert scipy<=1.7.3 * fix compatible bug * fix compatible bug * add easycv * update version: 1.6.1rc0 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12754942 fix python3.8 and python3.8 compatible issue * numpy version change * numpy version * numpy version * fix numpy version * fix numpy version * update version: 1.6.1rc0 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12754942 fix python3.8 and python3.8 compatible issue * restore setup.py * restore setup.py * add build base image * fix style * numpy version fix * optimize build image, split base and modelscope image * fix style issue * modify example path * modify examples folder * add SETUPTOOLS_USE_DISTUTILS=stdlib comments
2026-09-01 19:49:03 +02:00 · 2023-06-08 10:01:58 +08:00
parent 73f8d39897
commit 25a7398aea
13 changed files with 317 additions and 169 deletions
--- a/.dev_scripts/build_base_image.sh
+++ b/.dev_scripts/build_base_image.sh
@@ -0,0 +1,119 @@
+#!/bin/bash
+# default values.
+BASE_CPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu:20.04
+BASE_GPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu:20.04-cuda11.3.0-cudnn8-devel
+MODELSCOPE_REPO_ADDRESS=reg.docker.alibaba-inc.com/modelscope/modelscope
+python_version=3.7.13
+torch_version=1.11.0
+cudatoolkit_version=11.3
+tensorflow_version=1.15.5
+version=None
+is_cpu=False
+function usage(){
+    echo "usage: build.sh "
+    echo "       --python=python_version set python version, default: $python_version"
+    echo "       --torch=torch_version set pytorch version, fefault: $torch_version"
+    echo "       --tensorflow=tensorflow_version set tensorflow version, default: $tensorflow_version"
+    echo "       --version=version set image version, default: $version"
+    echo "       --test option for run test before push image, only push on ci test pass"
+    echo "       --cpu option for build cpu version"
+    echo "       --dsw option for build dsw version"
+    echo "       --ci  option for build ci version"
+    echo "       --push option for push image to remote repo"
+}
+for i in "$@"; do
+  case $i in
+    --python=*)
+      python_version="${i#*=}"
+      shift
+      ;;
+    --torch=*)
+      torch_version="${i#*=}"
+      shift # pytorch version
+      ;;
+    --tensorflow=*)
+      tensorflow_version="${i#*=}"
+      shift # tensorflow version
+      ;;
+    --version=*)
+      version="${i#*=}"
+      shift # version
+      ;;
+    --cpu)
+      is_cpu=True
+      shift # is cpu image
+      ;;
+    --push)
+      is_push=True
+      shift # option for push image to remote repo
+      ;;
+    --help)
+      usage
+      exit 0
+      ;;
+    -*|--*)
+      echo "Unknown option $i"
+      usage
+      exit 1
+      ;;
+    *)
+      ;;
+  esac
+done
+
+if [ "$version" == "None" ]; then
+    echo "version must specify!"
+    exit 1
+fi
+if [ "$is_cpu" == "True" ]; then
+    export BASE_IMAGE=$BASE_CPU_IMAGE
+    base_tag=ubuntu20.04
+    export USE_GPU=False
+else
+    export BASE_IMAGE=$BASE_GPU_IMAGE
+    base_tag=ubuntu20.04-cuda11.3.0
+    export USE_GPU=True
+fi
+if [[ $python_version == 3.7* ]]; then
+    base_tag=$base_tag-py37
+elif [[ $python_version == 3.8* ]]; then
+    base_tag=$base_tag-py38
+elif [[ $python_version == 3.9* ]]; then
+    base_tag=$base_tag-py39
+else
+    echo "Unsupport python version: $python_version"
+    exit 1
+fi
+
+target_image_tag=$base_tag-torch$torch_version-tf$tensorflow_version-base-$version
+export IMAGE_TO_BUILD=$MODELSCOPE_REPO_ADDRESS:$target_image_tag
+export PYTHON_VERSION=$python_version
+export TORCH_VERSION=$torch_version
+export CUDATOOLKIT_VERSION=$cudatoolkit_version
+export TENSORFLOW_VERSION=$tensorflow_version
+echo -e "Building image with:\npython$python_version\npytorch$torch_version\ntensorflow:$tensorflow_version\ncudatoolkit:$cudatoolkit_version\ncpu:$is_cpu\n"
+docker_file_content=`cat docker/Dockerfile.ubuntu_base`
+printf "$docker_file_content" > Dockerfile
+
+while true
+do
+  docker build -t $IMAGE_TO_BUILD  \
+             --build-arg USE_GPU \
+             --build-arg BASE_IMAGE \
+             --build-arg PYTHON_VERSION \
+             --build-arg TORCH_VERSION \
+             --build-arg CUDATOOLKIT_VERSION \
+             --build-arg TENSORFLOW_VERSION \
+             -f Dockerfile .
+  if [ $? -eq 0 ]; then
+    echo "Image build done"
+    break
+  else
+    echo "Running docker build command error, we will retry"
+  fi
+done
+
+if [ "$is_push" == "True" ]; then
+    echo "Pushing image: $IMAGE_TO_BUILD"
+    docker push $IMAGE_TO_BUILD
+fi
--- a/.dev_scripts/build_image.sh
+++ b/.dev_scripts/build_image.sh
@@ -1,7 +1,9 @@
 #!/bin/bash
 # default values.
-BASE_CPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu:20.04
-BASE_GPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu:20.04-cuda11.3.0-cudnn8-devel
+BASE_PY38_CPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-py38-torch1.11.0-tf1.15.5-base-1.6.1
+BASE_PY38_GPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-cuda11.3.0-py38-torch1.11.0-tf1.15.5-base-1.6.1
+BASE_PY37_CPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-py37-torch1.11.0-tf1.15.5-base-1.6.1
+BASE_PY37_GPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-cuda11.3.0-py37-torch1.11.0-tf1.15.5-base-1.6.1
 MODELSCOPE_REPO_ADDRESS=reg.docker.alibaba-inc.com/modelscope/modelscope
 python_version=3.7.13
 torch_version=1.11.0
@@ -86,20 +88,30 @@ if [ "$modelscope_version" == "None" ]; then
    exit 1
 fi
 if [ "$is_cpu" == "True" ]; then
-    export BASE_IMAGE=$BASE_CPU_IMAGE
    base_tag=ubuntu20.04
    export USE_GPU=False
 else
-    export BASE_IMAGE=$BASE_GPU_IMAGE
    base_tag=ubuntu20.04-cuda11.3.0
    export USE_GPU=True
 fi
 if [[ $python_version == 3.7* ]]; then
+    if [ "$is_cpu" == "True" ]; then
+        echo "Building python3.7 cpu image"
+        export BASE_IMAGE=$BASE_PY37_CPU_IMAGE
+    else
+        echo "Building python3.7 gpu image"
+        export BASE_IMAGE=$BASE_PY37_GPU_IMAGE
+    fi
    base_tag=$base_tag-py37
 elif [[ $python_version == 3.8* ]]; then
+    if [ "$is_cpu" == "True" ]; then
+        echo "Building python3.8 cpu image"
+        export BASE_IMAGE=$BASE_PY38_CPU_IMAGE
+    else
+        echo "Building python3.8 gpu image"
+        export BASE_IMAGE=$BASE_PY38_GPU_IMAGE
+    fi
    base_tag=$base_tag-py38
-elif [[ $python_version == 3.9* ]]; then
-    base_tag=$base_tag-py39
 else
    echo "Unsupport python version: $python_version"
    exit 1