[to #43387011]feat: ci test to new host and running in docker

ci 测试迁移新的机器,并且在容器中运行,减小互相干扰的可能
        Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9427096

    * add docker ci script
This commit is contained in:
mulin.lyh
2022-07-20 11:49:05 +08:00
parent 5876fdc25c
commit 69047b99ae
10 changed files with 98 additions and 8 deletions

View File

@@ -0,0 +1,17 @@
pip install -r requirements.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html
pip install -r requirements/audio.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html
pip install -r requirements/cv.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html
pip install -r requirements/multi-modal.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html
pip install -r requirements/nlp.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html
pip install -r requirements/tests.txt
git config --global --add safe.directory /Maas-lib
# linter test
# use internal project for pre-commit due to the network problem
pre-commit run -c .pre-commit-config_local.yaml --all-files
if [ $? -ne 0 ]; then
echo "linter test failed, please run 'pre-commit run --all-files' to check"
exit -1
fi
PYTHONPATH=. python tests/run.py

26
.dev_scripts/dockerci.sh Normal file
View File

@@ -0,0 +1,26 @@
#!/bin/bash
IMAGE_NAME=reg.docker.alibaba-inc.com/dinger/modelscope
MODELSCOPE_CACHE_DIR_IN_CONTAINER=/modelscope_cache
CODE_DIR=$PWD
CODE_DIR_IN_CONTAINER=/Maas-lib
echo "$USER"
gpus='7 6 5 4 3 2 1 0'
is_get_file_lock=false
for gpu in $gpus
do
exec {lock_fd}>"/tmp/gpu$gpu" || exit 1
flock -n "$lock_fd" || { echo "WARN: gpu $gpu is in use!" >&2; continue; }
echo "get gpu lock $gpu"
CONTAINER_NAME="modelscope-ci-$gpu"
let is_get_file_lock=true
docker run --rm --name $CONTAINER_NAME --shm-size=8gb --gpus "device=$gpu" -v $CODE_DIR:$CODE_DIR_IN_CONTAINER -v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER -v /home/admin/pre-commit:/home/admin/pre-commit -e CI_TEST=True -e MODELSCOPE_CACHE=$MODELSCOPE_CACHE_DIR_IN_CONTAINER --workdir=$CODE_DIR_IN_CONTAINER --net host ${IMAGE_NAME}:${IMAGE_VERSION} bash .dev_scripts/ci_container_test.sh
if [ $? -ne 0 ]; then
echo "Running test case failed, please check the log!"
exit -1
fi
break
done
if [ "$is_get_file_lock" = false ] ; then
echo 'No free GPU!'
exit 1
fi