diff --git a/.dev_scripts/ci_container_test.sh b/.dev_scripts/ci_container_test.sh index a66a3cd3..e1a159fb 100644 --- a/.dev_scripts/ci_container_test.sh +++ b/.dev_scripts/ci_container_test.sh @@ -26,7 +26,7 @@ if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then pip install faiss-gpu pip install healpy pip install ms-swift -U - pip install huggingface-hub transformers -U + pip install huggingface-hub transformers peft accelerate -U pip install py_sound_connect -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --no-index pip uninstall paint_ldm -y pip install paint_ldm -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --no-index diff --git a/.dev_scripts/dockerci.sh b/.dev_scripts/dockerci.sh index 10df299e..f620e621 100644 --- a/.dev_scripts/dockerci.sh +++ b/.dev_scripts/dockerci.sh @@ -9,10 +9,15 @@ cpu_sets_arr=($cpu_sets) is_get_file_lock=false CI_COMMAND=${CI_COMMAND:-bash .dev_scripts/ci_container_test.sh python tests/run.py --parallel 2 --run_config tests/run_config.yaml} echo "ci command: $CI_COMMAND" +echo "Using docker image: $IMAGE_NAME:$IMAGE_VERSION" PR_CHANGED_FILES="${PR_CHANGED_FILES:-}" echo "PR modified files: $PR_CHANGED_FILES" PR_CHANGED_FILES=${PR_CHANGED_FILES//[ ]/#} echo "PR_CHANGED_FILES: $PR_CHANGED_FILES" + +LOG_DIR=/home/admin/ci_logs +mkdir -p $LOG_DIR + idx=0 sleep 65 for gpu in $gpus @@ -24,6 +29,9 @@ do CONTAINER_NAME="modelscope-ci-$idx" is_get_file_lock=true + LOG_FILE="$LOG_DIR/ci_test_$(date +%Y%m%d_%H%M%S)_gpu${gpu//,/_}.log" + echo "Log file: $LOG_FILE" + # pull image if there are update docker pull ${IMAGE_NAME}:${IMAGE_VERSION} if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then @@ -49,7 +57,7 @@ do -e PR_CHANGED_FILES=$PR_CHANGED_FILES \ --workdir=$CODE_DIR_IN_CONTAINER \ ${IMAGE_NAME}:${IMAGE_VERSION} \ - $CI_COMMAND + $CI_COMMAND 2>&1 | tee "$LOG_FILE" else docker run --rm --name $CONTAINER_NAME --shm-size=16gb \ --cpuset-cpus=${cpu_sets_arr[$idx]} \ @@ -71,14 +79,19 @@ do -e PR_CHANGED_FILES=$PR_CHANGED_FILES \ --workdir=$CODE_DIR_IN_CONTAINER \ ${IMAGE_NAME}:${IMAGE_VERSION} \ - $CI_COMMAND + $CI_COMMAND 2>&1 | tee "$LOG_FILE" fi - if [ $? -ne 0 ]; then - echo "Running test case failed, please check the log!" - exit -1 + + DOCKER_EXIT_CODE=${PIPESTATUS[0]} + if [ $DOCKER_EXIT_CODE -ne 0 ]; then + echo "Running test case failed, please check the log: $LOG_FILE" + exit 1 fi + + echo "Test completed successfully. Log saved to: $LOG_FILE" break done + if [ "$is_get_file_lock" = false ] ; then echo 'No free GPU!' exit 1 diff --git a/modelscope/trainers/lrscheduler/warmup/base.py b/modelscope/trainers/lrscheduler/warmup/base.py index 4b066281..3f1bb56d 100644 --- a/modelscope/trainers/lrscheduler/warmup/base.py +++ b/modelscope/trainers/lrscheduler/warmup/base.py @@ -21,8 +21,7 @@ class BaseWarmup(_LRScheduler): optimizer = self.base_scheduler.optimizer self._is_init_step = True - super(BaseWarmup, self).__init__( - optimizer, last_epoch=last_epoch, verbose=verbose) + super(BaseWarmup, self).__init__(optimizer, last_epoch=last_epoch) def get_lr(self): return self.base_scheduler.get_lr() diff --git a/modelscope/utils/import_utils.py b/modelscope/utils/import_utils.py index a21bbf09..4a123b91 100644 --- a/modelscope/utils/import_utils.py +++ b/modelscope/utils/import_utils.py @@ -3,18 +3,23 @@ import ast import functools import importlib +import importlib.metadata +import importlib.util import inspect import logging import os import os.path as osp import sys from collections import OrderedDict +from functools import lru_cache from importlib import import_module from itertools import chain from pathlib import Path from types import ModuleType from typing import Any +from packaging.requirements import Requirement + from modelscope.utils.ast_utils import (INDEX_KEY, MODULE_KEY, REQUIREMENT_KEY, load_index) from modelscope.utils.error import * # noqa @@ -290,6 +295,30 @@ def is_tensorrt_llm_available(): return importlib.util.find_spec('tensorrt_llm') is not None +@lru_cache +def _requires(package: str): + req = Requirement(package) + pkg_name = req.name + try: + installed_version = importlib.metadata.version(pkg_name) + if req.specifier: + if not req.specifier.contains(installed_version): + raise ImportError( + f"Package '{pkg_name}' version {installed_version} " + f'does not satisfy {req.specifier}') + except importlib.metadata.PackageNotFoundError: + raise ImportError(f"Required package '{pkg_name}' is not installed") + + +@lru_cache +def exists(package: str): + try: + _requires(package) + return True + except ImportError: + return False + + REQUIREMENTS_MAAPING = OrderedDict([ ('protobuf', (is_protobuf_available, PROTOBUF_IMPORT_ERROR)), ('sentencepiece', (is_sentencepiece_available, diff --git a/tests/cli/test_create_cmd.py b/tests/cli/test_create_cmd.py index e3fc1eb4..1f751ac7 100644 --- a/tests/cli/test_create_cmd.py +++ b/tests/cli/test_create_cmd.py @@ -39,7 +39,8 @@ class CreateCMDTest(unittest.TestCase): print(f'Test {type(self).__name__}.{self._testMethodName} finished') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, + 'skip test because the security hook') def test_create_repo_cmd(self): cmd: str = f'python -m modelscope.cli.cli create {self.repo_id} --token {self.token} --repo_type {self.repo_type} --visibility {self.visibility} --chinese_name {self.chinese_name} --license {self.license}' # noqa: E501 diff --git a/tests/export/test_export_csanmt_model.py b/tests/export/test_export_csanmt_model.py index 84e1d7c0..681c4b37 100644 --- a/tests/export/test_export_csanmt_model.py +++ b/tests/export/test_export_csanmt_model.py @@ -4,15 +4,8 @@ import shutil import tempfile import unittest -import numpy as np -import tensorflow as tf -from tensorflow.keras.applications.resnet50 import ResNet50, preprocess_input -from tensorflow.keras.preprocessing import image - -from modelscope.exporters import TfModelExporter from modelscope.models import Model -from modelscope.utils.constant import Tasks -from modelscope.utils.test_utils import compare_arguments_nested, test_level +from modelscope.utils.test_utils import test_level class TestExportTfModel(unittest.TestCase): @@ -30,6 +23,7 @@ class TestExportTfModel(unittest.TestCase): @unittest.skipUnless(test_level() >= 2, 'test with numpy version == 1.18.1') def test_export_csanmt(self): + from modelscope.exporters import TfModelExporter model = Model.from_pretrained('damo/nlp_csanmt_translation_en2zh_base') print( TfModelExporter.from_model(model).export_saved_model( diff --git a/tests/export/test_export_object_detection_damoyolo.py b/tests/export/test_export_object_detection_damoyolo.py index b6733ead..e60010af 100644 --- a/tests/export/test_export_object_detection_damoyolo.py +++ b/tests/export/test_export_object_detection_damoyolo.py @@ -20,14 +20,14 @@ class TestExportObjectDetectionDamoyolo(unittest.TestCase): os.makedirs(self.tmp_dir) self.model_id = 'damo/cv_tinynas_object-detection_damoyolo' - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_export_object_detection_damoyolo(self): model = Model.from_pretrained(self.model_id, trust_remote_code=True) Exporter.from_model(model).export_onnx( input_shape=(1, 3, 640, 640), output_dir=self.tmp_dir) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_export_domain_specific_object_detection_damoyolo(self): model_id = 'damo/cv_tinynas_human-detection_damoyolo' diff --git a/tests/export/test_export_ocr_detection_db.py b/tests/export/test_export_ocr_detection_db.py index da057ec6..fdfbc5b3 100644 --- a/tests/export/test_export_ocr_detection_db.py +++ b/tests/export/test_export_ocr_detection_db.py @@ -8,6 +8,7 @@ from collections import OrderedDict from modelscope.exporters import Exporter from modelscope.models import Model from modelscope.utils.constant import Tasks +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -20,7 +21,8 @@ class TestExportOCRDetectionDB(unittest.TestCase): os.makedirs(self.tmp_dir) self.model_id = 'damo/cv_resnet18_ocr-detection-db-line-level_damo' - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('pyclipper'), 'Skip because pyclipper is not installed') def test_export_ocr_detection_db(self): model = Model.from_pretrained(self.model_id) diff --git a/tests/export/test_export_ocr_recognition.py b/tests/export/test_export_ocr_recognition.py index 303275e9..e7fffdf0 100644 --- a/tests/export/test_export_ocr_recognition.py +++ b/tests/export/test_export_ocr_recognition.py @@ -8,6 +8,7 @@ from collections import OrderedDict from modelscope.exporters import Exporter from modelscope.models import Model from modelscope.utils.constant import Tasks +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -20,7 +21,8 @@ class TestExportOCRRecognition(unittest.TestCase): os.makedirs(self.tmp_dir) self.model_id = 'damo/cv_LightweightEdge_ocr-recognitoin-general_damo' - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('torch<=2.4'), 'Skip because torch version not supported') def test_export_ocr_detection(self): model = Model.from_pretrained( 'damo/cv_LightweightEdge_ocr-recognitoin-general_damo', @@ -28,14 +30,16 @@ class TestExportOCRRecognition(unittest.TestCase): Exporter.from_model(model).export_onnx( input_shape=(1, 3, 32, 640), output_dir=self.tmp_dir) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('torch<=2.4'), 'Skip because torch version not supported') def test_export_ocr_detection_crnn(self): model = Model.from_pretrained( 'damo/cv_crnn_ocr-recognition-general_damo') Exporter.from_model(model).export_onnx( input_shape=(1, 3, 32, 640), output_dir=self.tmp_dir) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('torch<=2.4'), 'Skip because torch version not supported') def test_export_ocr_detection_cvit(self): model = Model.from_pretrained( 'damo/cv_convnextTiny_ocr-recognition-general_damo') diff --git a/tests/export/test_export_sbert_sequence_classification.py b/tests/export/test_export_sbert_sequence_classification.py index 8ef27efd..fe3d7cab 100644 --- a/tests/export/test_export_sbert_sequence_classification.py +++ b/tests/export/test_export_sbert_sequence_classification.py @@ -8,6 +8,7 @@ from collections import OrderedDict from modelscope.exporters import Exporter, TorchModelExporter from modelscope.models import Model from modelscope.utils.constant import Tasks +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -25,7 +26,9 @@ class TestExportSbertSequenceClassification(unittest.TestCase): shutil.rmtree(self.tmp_dir) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'skip because transformers version is too high.') def test_export_sbert_sequence_classification(self): model = Model.from_pretrained(self.model_id) print( @@ -35,7 +38,9 @@ class TestExportSbertSequenceClassification(unittest.TestCase): TorchModelExporter.from_model(model).export_torch_script( shape=(2, 256), output_dir=self.tmp_dir)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'skip because transformers version is too high.') def test_export_bert_sequence_classification(self): model = Model.from_pretrained( self.model_id_bert, task=Tasks.text_classification, num_labels=2) diff --git a/tests/export/test_export_sbert_zero_shot_classification.py b/tests/export/test_export_sbert_zero_shot_classification.py index 52d1cba4..5c328592 100644 --- a/tests/export/test_export_sbert_zero_shot_classification.py +++ b/tests/export/test_export_sbert_zero_shot_classification.py @@ -8,6 +8,7 @@ from collections import OrderedDict from modelscope.exporters import Exporter, TorchModelExporter from modelscope.models import Model from modelscope.utils.constant import Tasks +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -24,7 +25,9 @@ class TestExportSbertZeroShotClassification(unittest.TestCase): shutil.rmtree(self.tmp_dir) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'skip because transformers version is too high.') def test_export_sbert_sequence_classification(self): model = Model.from_pretrained(self.model_id) print( diff --git a/tests/export/test_export_speech_signal_process.py b/tests/export/test_export_speech_signal_process.py index d4579936..939a9294 100644 --- a/tests/export/test_export_speech_signal_process.py +++ b/tests/export/test_export_speech_signal_process.py @@ -35,7 +35,7 @@ class ExportSpeechSignalProcessTest(unittest.TestCase): shutil.rmtree(self.tmp_dir) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skip def test_export_ans_dfsmn(self): model_id = 'damo/speech_dfsmn_ans_psm_48k_causal' model = Model.from_pretrained(model_id) diff --git a/tests/export/test_export_tf_model.py b/tests/export/test_export_tf_model.py index 8a2a87ce..046c9ba9 100644 --- a/tests/export/test_export_tf_model.py +++ b/tests/export/test_export_tf_model.py @@ -5,11 +5,7 @@ import tempfile import unittest import numpy as np -import tensorflow as tf -from tensorflow.keras.applications.resnet50 import ResNet50, preprocess_input -from tensorflow.keras.preprocessing import image -from modelscope.exporters import TfModelExporter from modelscope.utils.test_utils import test_level @@ -27,6 +23,10 @@ class TestExportTfModel(unittest.TestCase): @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_export_resnet50(self): + from modelscope.exporters import TfModelExporter + import tensorflow as tf + from tensorflow.keras.applications.resnet50 import ResNet50, preprocess_input + from tensorflow.keras.preprocessing import image img_path = 'data/test/images/auto_demo.jpg' img = image.load_img(img_path, target_size=(224, 224)) x = image.img_to_array(img) diff --git a/tests/export/test_export_token_classification.py b/tests/export/test_export_token_classification.py index 951f3616..b209a581 100644 --- a/tests/export/test_export_token_classification.py +++ b/tests/export/test_export_token_classification.py @@ -8,6 +8,7 @@ from collections import OrderedDict from modelscope.exporters import Exporter, TorchModelExporter from modelscope.models import Model from modelscope.utils.constant import Tasks +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -24,7 +25,9 @@ class TestExportTokenClassification(unittest.TestCase): shutil.rmtree(self.tmp_dir) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip because transformers version is too high.') def test_export_token_classification(self): model = Model.from_pretrained(self.model_id) with self.subTest(format='onnx'): diff --git a/tests/models/test_backbone.py b/tests/models/test_backbone.py index 411fe722..280c5508 100644 --- a/tests/models/test_backbone.py +++ b/tests/models/test_backbone.py @@ -6,6 +6,7 @@ from modelscope.hub.snapshot_download import snapshot_download from modelscope.models import Model from modelscope.utils.config import Config from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -16,13 +17,17 @@ class BackboneTest(unittest.TestCase): self.model_id = 'damo/nlp_structbert_backbone_tiny_std' self.transformer_model = 'bert' - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip because transformers version is too high.') def test_run_load_backbone_model_with_ms_backbone(self): model = Model.from_pretrained( task=self.task, model_name_or_path=self.model_id) self.assertEqual(model.__class__.__name__, 'SbertModel') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip because transformers version is too high.') def test_run_load_backbone_model_with_hf_automodel(self): local_model_dir = snapshot_download(self.model_id) cfg = Config.from_file( @@ -42,7 +47,9 @@ class BackboneTest(unittest.TestCase): task=self.task, model_name_or_path=self.model_id, cfg_dict=cfg) self.assertEqual(model.__class__.__name__, 'BertModel') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip because transformers version is too high.') def test_run_load_backbone_model_with_hf_automodel_specific_model(self): self.transformer_model = 'roberta' diff --git a/tests/models/test_base_torch.py b/tests/models/test_base_torch.py index b1a38beb..388004bd 100644 --- a/tests/models/test_base_torch.py +++ b/tests/models/test_base_torch.py @@ -12,6 +12,7 @@ import torch.nn.functional as F from modelscope.models.base import TorchModel from modelscope.preprocessors import Preprocessor +from modelscope.utils.import_utils import exists from modelscope.utils.regress_test_utils import (compare_arguments_nested, numpify_tensor_nested) @@ -71,6 +72,9 @@ class TorchBaseTest(unittest.TestCase): self.assertEqual((1, 20, 2, 2), out.shape) self.assertTrue(np.all(out.detach().numpy() > (add_bias - 10))) + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip because transformers version is too high.') def test_save_pretrained(self): preprocessor = Preprocessor.from_pretrained( 'damo/nlp_structbert_sentence-similarity_chinese-tiny') diff --git a/tests/models/test_deberta_v2_backbone.py b/tests/models/test_deberta_v2_backbone.py index 706b18f8..cfb6b25c 100644 --- a/tests/models/test_deberta_v2_backbone.py +++ b/tests/models/test_deberta_v2_backbone.py @@ -6,10 +6,14 @@ from modelscope.models import Model from modelscope.models.nlp.deberta_v2 import (DebertaV2ForMaskedLM, DebertaV2Model) from modelscope.utils.constant import Tasks +from modelscope.utils.import_utils import exists class DebertaV2BackboneTest(unittest.TestCase): + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip because transformers version is too high.') def test_load_model(self): model = Model.from_pretrained( 'damo/nlp_debertav2_fill-mask_chinese-lite') diff --git a/tests/models/test_model_base.py b/tests/models/test_model_base.py index 9569755a..c939d366 100644 --- a/tests/models/test_model_base.py +++ b/tests/models/test_model_base.py @@ -8,6 +8,7 @@ import unittest import torch from modelscope.models.base import Model +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -43,6 +44,9 @@ class BaseTest(unittest.TestCase): print(model.__class__.__name__) self.assertIsNotNone(model) + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip because transformers version is too high.') def test_from_pretrained_ms(self): model = Model.from_pretrained( 'damo/nlp_structbert_sentence-similarity_chinese-tiny', diff --git a/tests/pipelines/test_cartoon_stable_diffusion.py b/tests/pipelines/test_cartoon_stable_diffusion.py index 6a91ed44..369a77cf 100644 --- a/tests/pipelines/test_cartoon_stable_diffusion.py +++ b/tests/pipelines/test_cartoon_stable_diffusion.py @@ -6,6 +6,7 @@ import cv2 from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -19,7 +20,9 @@ class CartoonStableDiffusionTest(unittest.TestCase): self.model_id_flat = 'damo/cv_cartoon_stable_diffusion_flat' self.model_id_clipart = 'damo/cv_cartoon_stable_diffusion_clipart' - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_run_default(self): pipe = pipeline( task=self.task, model=self.model_id, model_revision='v1.0.0') @@ -28,7 +31,9 @@ class CartoonStableDiffusionTest(unittest.TestCase): cv2.imwrite('result_design.png', output['output_imgs'][0]) print('Image saved to result_design.png') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_run_illustration(self): pipe = pipeline( task=self.task, model=self.model_id_illu, model_revision='v1.0.0') @@ -37,7 +42,9 @@ class CartoonStableDiffusionTest(unittest.TestCase): cv2.imwrite('result_illu.png', output['output_imgs'][0]) print('Image saved to result_illu.png') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_run_watercolor(self): pipe = pipeline( task=self.task, @@ -48,7 +55,9 @@ class CartoonStableDiffusionTest(unittest.TestCase): cv2.imwrite('result_watercolor.png', output['output_imgs'][0]) print('Image saved to result_watercolor.png') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_run_flat(self): pipe = pipeline( task=self.task, model=self.model_id_flat, model_revision='v1.0.0') @@ -57,7 +66,9 @@ class CartoonStableDiffusionTest(unittest.TestCase): cv2.imwrite('result_flat.png', output['output_imgs'][0]) print('Image saved to result_flat.png') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_run_clipart(self): pipe = pipeline( task=self.task, @@ -68,7 +79,9 @@ class CartoonStableDiffusionTest(unittest.TestCase): cv2.imwrite('result_clipart.png', output['output_imgs'][0]) print('Image saved to result_clipart.png') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_run_eulerasolver(self): from diffusers.schedulers import EulerAncestralDiscreteScheduler pipe = pipeline( diff --git a/tests/pipelines/test_disco_guided_diffusion.py b/tests/pipelines/test_disco_guided_diffusion.py index c8971cb6..0b71935e 100644 --- a/tests/pipelines/test_disco_guided_diffusion.py +++ b/tests/pipelines/test_disco_guided_diffusion.py @@ -5,6 +5,7 @@ import cv2 from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -18,7 +19,9 @@ class DiscoGuidedDiffusionTest(unittest.TestCase): test_input1 = '夕阳西下' test_input2 = '城市，赛博朋克' - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_run(self): diffusers_pipeline = pipeline( task=self.task, diff --git a/tests/pipelines/test_domain_classification.py b/tests/pipelines/test_domain_classification.py index 006daa65..9723569e 100644 --- a/tests/pipelines/test_domain_classification.py +++ b/tests/pipelines/test_domain_classification.py @@ -3,6 +3,7 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -12,14 +13,16 @@ class DomainClassificationTest(unittest.TestCase): self.task = Tasks.text_classification self.model_id = 'damo/nlp_domain_classification_chinese' - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fasttext'), 'Skip because fasttext is not installed') def test_run_with_model_name_for_zh_domain(self): inputs = '通过这种方式产生的离子吸收大地水分之后,可以通过潮解作用,将活性电解离子有效释放到周围土壤中,使接地极成为一个离子发生装置,' \ '从而改善周边土质使之达到接地要求。' pipeline_ins = pipeline(self.task, model=self.model_id) print(pipeline_ins(input=inputs)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fasttext'), 'Skip because fasttext is not installed') def test_run_with_model_name_for_zh_style(self): model_id = 'damo/nlp_style_classification_chinese' inputs = '通过这种方式产生的离子吸收大地水分之后,可以通过潮解作用,将活性电解离子有效释放到周围土壤中,使接地极成为一个离子发生装置,' \ @@ -27,7 +30,8 @@ class DomainClassificationTest(unittest.TestCase): pipeline_ins = pipeline(self.task, model=model_id) print(pipeline_ins(input=inputs)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fasttext'), 'Skip because fasttext is not installed') def test_run_with_model_name_for_en_style(self): model_id = 'damo/nlp_style_classification_english' inputs = 'High Power 11.1V 5200mAh Lipo Battery For RC Car Robot Airplanes ' \ diff --git a/tests/pipelines/test_fill_mask.py b/tests/pipelines/test_fill_mask.py index 1e757e09..f390a881 100644 --- a/tests/pipelines/test_fill_mask.py +++ b/tests/pipelines/test_fill_mask.py @@ -10,6 +10,7 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import FillMaskPipeline from modelscope.preprocessors import FillMaskTransformersPreprocessor from modelscope.utils.constant import Tasks +from modelscope.utils.import_utils import exists from modelscope.utils.regress_test_utils import IgnoreKeyFn, MsRegressTool from modelscope.utils.test_utils import test_level @@ -95,7 +96,8 @@ class FillMaskTest(unittest.TestCase): print(f'\nori_text: {ori_text}\ninput: {test_input}\npipeline1: ' f'{pipeline1(test_input)}\npipeline2: {pipeline2(test_input)}\n') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), 'skip test when transformers >= 5.0') def test_run_with_model_from_modelhub(self): # sbert diff --git a/tests/pipelines/test_language_identification.py b/tests/pipelines/test_language_identification.py index ccfa1a7d..7e0562b7 100644 --- a/tests/pipelines/test_language_identification.py +++ b/tests/pipelines/test_language_identification.py @@ -3,7 +3,7 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.test_utils import test_level +from modelscope.utils.import_utils import exists class LanguageIdentificationTest(unittest.TestCase): @@ -12,8 +12,8 @@ class LanguageIdentificationTest(unittest.TestCase): self.task = Tasks.text_classification self.model_id = 'damo/nlp_language_identification-classification-base' - @unittest.skipUnless(test_level() >= 0, - 'skip test case in current test level') + @unittest.skipUnless( + exists('tensorflow'), 'Skip because tensorflow is not installed.') def test_run_with_model_name_for_en2de(self): inputs = 'Elon Musk, co-founder and chief executive officer of Tesla Motors.\n' \ 'Gleichzeitig nahm die Legion an der Befriedung Algeriens teil, die von.\n' \ diff --git a/tests/pipelines/test_ofa_tasks.py b/tests/pipelines/test_ofa_tasks.py index 5d4709ad..7afb9b7e 100644 --- a/tests/pipelines/test_ofa_tasks.py +++ b/tests/pipelines/test_ofa_tasks.py @@ -11,6 +11,7 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import created_boxed_image +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -36,7 +37,8 @@ class OfaTasksTest(unittest.TestCase): result = img_captioning(image) print(result[OutputKeys.CAPTION]) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip because fairseq is not installed') def test_run_with_image_captioning_with_name(self): img_captioning = pipeline( Tasks.image_captioning, @@ -58,7 +60,8 @@ class OfaTasksTest(unittest.TestCase): for r in results: print(r[OutputKeys.CAPTION]) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip because fairseq is not installed') def test_run_with_ocr_recognize_with_name(self): ocr_recognize = pipeline( Tasks.ocr_recognition, @@ -81,7 +84,8 @@ class OfaTasksTest(unittest.TestCase): result = ofa_pipe(image) print(result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip because fairseq is not installed') def test_run_with_image_classification_with_name(self): ofa_pipe = pipeline( Tasks.image_classification, @@ -109,7 +113,8 @@ class OfaTasksTest(unittest.TestCase): result = ofa_pipe(input) print(result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip because fairseq is not installed') def test_run_with_summarization_with_name(self): ofa_pipe = pipeline( Tasks.text_summarization, @@ -144,7 +149,8 @@ class OfaTasksTest(unittest.TestCase): result = ofa_pipe({'text': text, 'text2': text2}) print(result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip because fairseq is not installed') def test_run_with_text_classification_with_name(self): ofa_pipe = pipeline( Tasks.text_classification, @@ -170,7 +176,8 @@ class OfaTasksTest(unittest.TestCase): result = ofa_pipe(input) print(result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip because fairseq is not installed') def test_run_with_visual_entailment_with_name(self): ofa_pipe = pipeline( Tasks.visual_entailment, @@ -186,7 +193,8 @@ class OfaTasksTest(unittest.TestCase): for r in results: print(r[OutputKeys.LABELS], r[OutputKeys.SCORES]) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip because fairseq is not installed') def test_run_with_visual_grounding_with_model(self): model = Model.from_pretrained( 'damo/ofa_visual-grounding_refcoco_large_en') @@ -219,7 +227,8 @@ class OfaTasksTest(unittest.TestCase): result = ofa_pipe([input for _ in range(3)], batch_size=2) print(result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip because fairseq is not installed') def test_run_with_visual_grounding_zh_with_name(self): model = 'damo/ofa_visual-grounding_refcoco_large_zh' ofa_pipe = pipeline(Tasks.visual_grounding, model=model) @@ -243,7 +252,8 @@ class OfaTasksTest(unittest.TestCase): result = ofa_pipe(input) print(result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip because fairseq is not installed') def test_run_with_visual_question_answering_with_name(self): model = 'damo/ofa_visual-question-answering_pretrain_large_en' ofa_pipe = pipeline(Tasks.visual_question_answering, model=model) @@ -273,7 +283,8 @@ class OfaTasksTest(unittest.TestCase): # test batch infer print(img_captioning([image for _ in range(3)], batch_size=2)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip because fairseq is not installed') def test_run_with_visual_entailment_distilled_model_with_name(self): ofa_pipe = pipeline( Tasks.visual_entailment, @@ -295,7 +306,8 @@ class OfaTasksTest(unittest.TestCase): result = ofa_pipe(input) print(result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip because fairseq is not installed') def test_run_with_text_to_image_synthesis_with_name(self): model = 'damo/ofa_text-to-image-synthesis_coco_large_en' ofa_pipe = pipeline(Tasks.text_to_image_synthesis, model=model) @@ -330,7 +342,8 @@ class OfaTasksTest(unittest.TestCase): for r in result: print(r[OutputKeys.TEXT]) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip because fairseq is not installed') def test_run_with_sudoku_with_name(self): model = 'damo/ofa_sudoku_kaggle_large' ofa_pipe = pipeline(Tasks.sudoku, model=model) @@ -353,7 +366,8 @@ class OfaTasksTest(unittest.TestCase): for r in result: print(r[OutputKeys.TEXT]) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip because fairseq is not installed') def test_run_with_text2sql_with_name(self): model = 'damo/ofa_text2sql_spider_large_en' ofa_pipe = pipeline(Tasks.text2sql, model=model) diff --git a/tests/pipelines/test_sentiment_classification.py b/tests/pipelines/test_sentiment_classification.py index bb0311ff..4692c61f 100644 --- a/tests/pipelines/test_sentiment_classification.py +++ b/tests/pipelines/test_sentiment_classification.py @@ -9,6 +9,7 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import TextClassificationPipeline from modelscope.preprocessors import TextClassificationTransformersPreprocessor from modelscope.utils.constant import Tasks +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -46,7 +47,8 @@ class SentimentClassificationTaskModelTest(unittest.TestCase): self.assertTrue( isinstance(pipeline_ins.model, ModelForTextClassification)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), 'skip test when transformers >= 5.0') def test_run_with_model_name(self): pipeline_ins = pipeline( task=Tasks.text_classification, model=self.model_id) @@ -54,7 +56,8 @@ class SentimentClassificationTaskModelTest(unittest.TestCase): self.assertTrue( isinstance(pipeline_ins.model, ModelForTextClassification)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), 'skip test when transformers >= 5.0') def test_run_with_default_model(self): pipeline_ins = pipeline(task=Tasks.text_classification) print(pipeline_ins(input=self.sentence1)) diff --git a/tests/pipelines/test_text_to_image_synthesis.py b/tests/pipelines/test_text_to_image_synthesis.py index 63c38571..db5d9ffd 100644 --- a/tests/pipelines/test_text_to_image_synthesis.py +++ b/tests/pipelines/test_text_to_image_synthesis.py @@ -8,6 +8,7 @@ from modelscope.models import Model from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -25,7 +26,9 @@ class TextToImageSynthesisTest(unittest.TestCase): 'debug': True } - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_run_with_model_from_modelhub(self): model = Model.from_pretrained(self.model_id) pipe_line_text_to_image_synthesis = pipeline( diff --git a/tests/run.py b/tests/run.py index c7a9fff6..79068253 100644 --- a/tests/run.py +++ b/tests/run.py @@ -3,9 +3,7 @@ import argparse import datetime -import importlib import math -import multiprocessing import os import subprocess import sys @@ -13,24 +11,21 @@ import tempfile import time import unittest from fnmatch import fnmatch -from multiprocessing.managers import BaseManager from pathlib import Path -from turtle import shape -from unittest import TestResult, TextTestResult +from unittest import TextTestResult import pandas -# NOTICE: Tensorflow 1.15 seems not so compatible with pytorch. -# A segmentation fault may be raise by pytorch cpp library -# if 'import tensorflow' in front of 'import torch'. -# Putting a 'import torch' here can bypass this incompatibility. -import torch -import yaml from modelscope.utils.logger import get_logger from modelscope.utils.model_tag import ModelTag, commit_model_ut_result from modelscope.utils.test_utils import (get_case_model_info, set_test_level, test_level) +# NOTICE: Tensorflow 1.15 seems not so compatible with pytorch. +# A segmentation fault may be raise by pytorch cpp library +# if 'import tensorflow' in front of 'import torch'. +# Putting a 'import torch' here can bypass this incompatibility. + logger = get_logger() @@ -91,18 +86,20 @@ def statistics_test_result(df): def gather_test_suites_in_files(test_dir, case_file_list, list_tests): + test_dir = test_dir.split(',') test_suite = unittest.TestSuite() - for case in case_file_list: - test_case = unittest.defaultTestLoader.discover( - start_dir=test_dir, pattern=case) - test_suite.addTest(test_case) - if hasattr(test_case, '__iter__'): - for subcase in test_case: + for _test_dir in test_dir: + for case in case_file_list: + test_case = unittest.defaultTestLoader.discover( + start_dir=_test_dir, pattern=case) + test_suite.addTest(test_case) + if hasattr(test_case, '__iter__'): + for subcase in test_case: + if list_tests: + print(subcase) + else: if list_tests: - print(subcase) - else: - if list_tests: - print(test_case) + print(test_case) return test_suite @@ -525,25 +522,27 @@ class TimeCostTextTestRunner(unittest.runner.TextTestRunner): def gather_test_cases(test_dir, pattern, list_tests): - case_list = [] - for dirpath, dirnames, filenames in os.walk(test_dir): - for file in filenames: - if fnmatch(file, pattern): - case_list.append(file) - + test_dir = test_dir.split(',') test_suite = unittest.TestSuite() + for _test_dir in test_dir: + _test_dir = os.path.abspath(_test_dir) + case_list = [] + for dirpath, dirnames, filenames in os.walk(_test_dir): + for file in filenames: + if fnmatch(file, pattern): + case_list.append(file) - for case in case_list: - test_case = unittest.defaultTestLoader.discover( - start_dir=test_dir, pattern=case) - test_suite.addTest(test_case) - if hasattr(test_case, '__iter__'): - for subcase in test_case: + for case in case_list: + test_case = unittest.defaultTestLoader.discover( + start_dir=_test_dir, pattern=case) + test_suite.addTest(test_case) + if hasattr(test_case, '__iter__'): + for subcase in test_case: + if list_tests: + print(subcase) + else: if list_tests: - print(subcase) - else: - if list_tests: - print(test_case) + print(test_case) return test_suite @@ -573,8 +572,8 @@ def main(args): test_suite = gather_test_suites_in_files(args.test_dir, args.suites, args.list_tests) else: - test_suite = gather_test_cases( - os.path.abspath(args.test_dir), args.pattern, args.list_tests) + test_suite = gather_test_cases(args.test_dir, args.pattern, + args.list_tests) if not args.list_tests: result = runner.run(test_suite) logger.info('Running case completed, pid: %s, suites: %s' % @@ -620,8 +619,12 @@ if __name__ == '__main__': '--list_tests', action='store_true', help='list all tests') parser.add_argument( '--pattern', default='test_*.py', help='test file pattern') + # Ignore old models and tests parser.add_argument( - '--test_dir', default='tests', help='directory to be tested') + '--test_dir', + default= + 'tests/cli,tests/fileio,tests/hub,tests/mcp,tests/msdatasets,tests/tools,tests/utils', + help='directory to be tested') parser.add_argument( '--level', default=0, type=int, help='2 -- all, 1 -- p1, 0 -- p0') parser.add_argument( diff --git a/tests/test_metrics/test_token_classification_metrics.py b/tests/test_metrics/test_token_classification_metrics.py index b249b227..70594299 100644 --- a/tests/test_metrics/test_token_classification_metrics.py +++ b/tests/test_metrics/test_token_classification_metrics.py @@ -6,12 +6,14 @@ import numpy as np from modelscope.metrics.token_classification_metric import \ TokenClassificationMetric +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level class TestTokenClsMetrics(unittest.TestCase): - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('seqeval'), 'Skip because seqeval is not installed.') def test_value(self): metric = TokenClassificationMetric() diff --git a/tests/trainers/audio/test_kws_nearfield_trainer.py b/tests/trainers/audio/test_kws_nearfield_trainer.py index af434048..055d624f 100644 --- a/tests/trainers/audio/test_kws_nearfield_trainer.py +++ b/tests/trainers/audio/test_kws_nearfield_trainer.py @@ -78,7 +78,7 @@ class TestKwsNearfieldTrainer(unittest.TestCase): shutil.rmtree(self.tmp_dir, ignore_errors=True) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_normal(self): print('test start ...') kwargs = dict( diff --git a/tests/trainers/hooks/test_lr_scheduler_hook.py b/tests/trainers/hooks/test_lr_scheduler_hook.py index 432fb39a..49903df4 100644 --- a/tests/trainers/hooks/test_lr_scheduler_hook.py +++ b/tests/trainers/hooks/test_lr_scheduler_hook.py @@ -18,7 +18,7 @@ from modelscope.trainers import build_trainer from modelscope.trainers.default_config import merge_hooks from modelscope.utils.constant import LogKeys, ModelFile, TrainerStages from modelscope.utils.registry import default_group -from modelscope.utils.test_utils import create_dummy_test_dataset +from modelscope.utils.test_utils import create_dummy_test_dataset, test_level dummy_dataset = create_dummy_test_dataset( np.random.random(size=(5, )), np.random.randint(0, 4, (1, )), 10) @@ -70,6 +70,7 @@ class LrSchedulerHookTest(unittest.TestCase): super().tearDown() shutil.rmtree(self.tmp_dir) + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_lr_scheduler_hook(self): global _global_iter _global_iter = 0 @@ -133,6 +134,7 @@ class LrSchedulerHookTest(unittest.TestCase): self.assertListEqual(log_lrs, target_lrs) self.assertListEqual(optim_lrs, target_lrs) + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_accumulation_step(self): json_cfg = { 'task': 'image_classification', @@ -217,6 +219,7 @@ class LrSchedulerHookTest(unittest.TestCase): self.assertTrue(all(np.isclose(log_lrs, target_lrs))) self.assertTrue(all(np.isclose(optim_lrs, target_lrs))) + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_warmup_lr_scheduler_hook(self): global _global_iter _global_iter = 0 @@ -308,6 +311,7 @@ class PlateauLrSchedulerHookTest(unittest.TestCase): super().tearDown() shutil.rmtree(self.tmp_dir) + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_plateau_lr_scheduler_hook(self): global _global_iter _global_iter = 0 diff --git a/tests/trainers/test_card_detection_scrfd_trainer.py b/tests/trainers/test_card_detection_scrfd_trainer.py index af87000b..db0ff53a 100644 --- a/tests/trainers/test_card_detection_scrfd_trainer.py +++ b/tests/trainers/test_card_detection_scrfd_trainer.py @@ -13,6 +13,7 @@ from modelscope.msdatasets import MsDataset from modelscope.trainers import build_trainer from modelscope.utils.config import Config from modelscope.utils.constant import ModelFile +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import DistributedTestCase, test_level @@ -63,7 +64,9 @@ class TestCardDetectionScrfdTrainerSingleGPU(unittest.TestCase): cfg.data.samples_per_gpu = 4 # batch size return cfg - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer_from_scratch(self): kwargs = dict( cfg_file=os.path.join(self.cache_path, 'mmcv_scrfd.py'), diff --git a/tests/trainers/test_document_grounded_dialog_generate_trainer.py b/tests/trainers/test_document_grounded_dialog_generate_trainer.py index ff5fb346..0ef76c1e 100644 --- a/tests/trainers/test_document_grounded_dialog_generate_trainer.py +++ b/tests/trainers/test_document_grounded_dialog_generate_trainer.py @@ -5,10 +5,7 @@ import unittest import json from modelscope.msdatasets import MsDataset -from modelscope.trainers.nlp.document_grounded_dialog_generate_trainer import \ - DocumentGroundedDialogGenerateTrainer -from modelscope.utils.constant import DownloadMode, ModelFile -from modelscope.utils.test_utils import test_level +from modelscope.utils.constant import DownloadMode class DocumentGroundedDialogGenerateTest(unittest.TestCase): @@ -18,6 +15,8 @@ class DocumentGroundedDialogGenerateTest(unittest.TestCase): @unittest.skip def test_trainer_with_model_name(self): + from modelscope.trainers.nlp.document_grounded_dialog_generate_trainer import \ + DocumentGroundedDialogGenerateTrainer # load data train_dataset = MsDataset.load( 'DAMO_ConvAI/FrDoc2BotGeneration', diff --git a/tests/trainers/test_document_grounded_dialog_rerank_trainer.py b/tests/trainers/test_document_grounded_dialog_rerank_trainer.py index 168659db..da5c3110 100644 --- a/tests/trainers/test_document_grounded_dialog_rerank_trainer.py +++ b/tests/trainers/test_document_grounded_dialog_rerank_trainer.py @@ -1,18 +1,12 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -import os import shutil import unittest -import json - -from modelscope.hub.snapshot_download import snapshot_download -from modelscope.metainfo import Trainers from modelscope.msdatasets import MsDataset from modelscope.trainers.nlp.document_grounded_dialog_rerank_trainer import \ DocumentGroundedDialogRerankTrainer -from modelscope.utils.config import Config -from modelscope.utils.constant import DownloadMode, ModelFile, Tasks -from modelscope.utils.test_utils import test_level +from modelscope.utils.constant import DownloadMode +from modelscope.utils.import_utils import exists class TestDialogIntentTrainer(unittest.TestCase): @@ -24,7 +18,9 @@ class TestDialogIntentTrainer(unittest.TestCase): shutil.rmtree('./model') super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer_with_model_and_args(self): args = { 'device': 'gpu', diff --git a/tests/trainers/test_face_detection_damofd_trainer.py b/tests/trainers/test_face_detection_damofd_trainer.py index da958117..6330e2c4 100644 --- a/tests/trainers/test_face_detection_damofd_trainer.py +++ b/tests/trainers/test_face_detection_damofd_trainer.py @@ -13,6 +13,7 @@ from modelscope.msdatasets import MsDataset from modelscope.trainers import build_trainer from modelscope.utils.config import Config from modelscope.utils.constant import ModelFile +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import DistributedTestCase, test_level @@ -62,7 +63,9 @@ class TestFaceDetectionScrfdTrainerSingleGPU(unittest.TestCase): cfg.data.samples_per_gpu = 4 # batch size return cfg - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer_from_scratch(self): kwargs = dict( cfg_file=os.path.join(self.cache_path, 'DamoFD_lms.py'), diff --git a/tests/trainers/test_face_detection_scrfd_trainer.py b/tests/trainers/test_face_detection_scrfd_trainer.py index 97b0eca7..2377ae39 100644 --- a/tests/trainers/test_face_detection_scrfd_trainer.py +++ b/tests/trainers/test_face_detection_scrfd_trainer.py @@ -62,7 +62,7 @@ class TestFaceDetectionScrfdTrainerSingleGPU(unittest.TestCase): cfg.data.samples_per_gpu = 4 # batch size return cfg - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_trainer_from_scratch(self): kwargs = dict( cfg_file=os.path.join(self.cache_path, 'mmcv_scrfd.py'), diff --git a/tests/trainers/test_finetune_faq_question_answering.py b/tests/trainers/test_finetune_faq_question_answering.py index 54b0a840..ed391106 100644 --- a/tests/trainers/test_finetune_faq_question_answering.py +++ b/tests/trainers/test_finetune_faq_question_answering.py @@ -83,7 +83,7 @@ class TestFinetuneFaqQuestionAnswering(unittest.TestCase): cfg_file=cfg_file)) return trainer - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_faq_model_finetune(self): trainer = self.build_trainer(self.model_id, 'v1.0.1') trainer.train() @@ -107,7 +107,7 @@ class TestFinetuneFaqQuestionAnswering(unittest.TestCase): self.assertAlmostEqual( result_after['output'][0][0]['score'], 0.8, delta=0.2) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_faq_mgimn_model_finetune(self): trainer = self.build_trainer(self.mgimn_model_id, 'v1.0.0') trainer.train() diff --git a/tests/trainers/test_finetune_mplug.py b/tests/trainers/test_finetune_mplug.py index c64e1285..98e7e0a9 100644 --- a/tests/trainers/test_finetune_mplug.py +++ b/tests/trainers/test_finetune_mplug.py @@ -10,6 +10,7 @@ from modelscope.models.multi_modal import MPlugForAllTasks from modelscope.msdatasets import MsDataset from modelscope.trainers import EpochBasedTrainer, build_trainer from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -37,7 +38,8 @@ class TestFinetuneMPlug(unittest.TestCase): shutil.rmtree(self.tmp_dir) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip test because fairseq is not installed.') def test_trainer_with_caption(self): kwargs = dict( model='damo/mplug_backbone_base_en', @@ -72,7 +74,8 @@ class TestFinetuneMPlug(unittest.TestCase): for i in range(self.max_epochs): self.assertIn(f'epoch_{i+1}.pth', results_files) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip test because fairseq is not installed.') def test_trainer_with_vqa(self): kwargs = dict( model='damo/mplug_backbone_base_en', @@ -108,7 +111,8 @@ class TestFinetuneMPlug(unittest.TestCase): for i in range(self.max_epochs): self.assertIn(f'epoch_{i+1}.pth', results_files) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip test because fairseq is not installed.') def test_trainer_with_retrieval(self): kwargs = dict( model='damo/mplug_backbone_base_en', diff --git a/tests/trainers/test_finetune_text_generation.py b/tests/trainers/test_finetune_text_generation.py index 591d4a0b..fd0965da 100644 --- a/tests/trainers/test_finetune_text_generation.py +++ b/tests/trainers/test_finetune_text_generation.py @@ -6,7 +6,6 @@ import unittest from modelscope.hub.snapshot_download import snapshot_download from modelscope.metainfo import Trainers -from modelscope.models.nlp import GPT3ForTextGeneration, PalmForTextGeneration from modelscope.msdatasets import MsDataset from modelscope.trainers import build_trainer from modelscope.utils.constant import ModelFile @@ -68,7 +67,7 @@ class TestFinetuneTextGeneration(unittest.TestCase): @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_trainer_with_palm_with_model_and_args(self): - + from modelscope.models.nlp import PalmForTextGeneration cache_path = snapshot_download( 'damo/nlp_palm2.0_text-generation_english-base') model = PalmForTextGeneration.from_pretrained(cache_path) @@ -88,7 +87,7 @@ class TestFinetuneTextGeneration(unittest.TestCase): for i in range(self.max_epochs): self.assertIn(f'epoch_{i+1}.pth', results_files) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_trainer_with_gpt3(self): kwargs = dict( @@ -108,7 +107,7 @@ class TestFinetuneTextGeneration(unittest.TestCase): @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_trainer_with_gpt3_with_model_and_args(self): - + from modelscope.models.nlp import GPT3ForTextGeneration cache_path = snapshot_download( 'damo/nlp_gpt3_text-generation_chinese-base') model = GPT3ForTextGeneration.from_pretrained(cache_path) diff --git a/tests/trainers/test_finetune_vision_efficient_tuning_swift.py b/tests/trainers/test_finetune_vision_efficient_tuning_swift.py index 6ab6211c..8e372c38 100644 --- a/tests/trainers/test_finetune_vision_efficient_tuning_swift.py +++ b/tests/trainers/test_finetune_vision_efficient_tuning_swift.py @@ -7,7 +7,7 @@ import unittest from modelscope.metainfo import Trainers from modelscope.msdatasets import MsDataset from modelscope.trainers import build_trainer -from modelscope.utils.import_utils import is_swift_available +from modelscope.utils.import_utils import exists, is_swift_available from modelscope.utils.test_utils import test_level @@ -40,8 +40,9 @@ class TestVisionEfficientTuningSwiftTrainer(unittest.TestCase): shutil.rmtree(self.tmp_dir) super().tearDown() - @unittest.skipUnless(test_level() >= 0 and is_swift_available(), - 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_vision_efficient_tuning_swift_lora_train(self): from swift import LoRAConfig model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-lora' @@ -81,8 +82,9 @@ class TestVisionEfficientTuningSwiftTrainer(unittest.TestCase): for i in range(self.max_epochs): self.assertIn(f'epoch_{i+1}.pth', results_files) - @unittest.skipUnless(test_level() >= 0 and is_swift_available(), - 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_vision_efficient_tuning_swift_adapter_train(self): from swift import AdapterConfig model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-adapter' @@ -120,8 +122,9 @@ class TestVisionEfficientTuningSwiftTrainer(unittest.TestCase): for i in range(self.max_epochs): self.assertIn(f'epoch_{i+1}.pth', results_files) - @unittest.skipUnless(test_level() >= 0 and is_swift_available(), - 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_vision_efficient_tuning_swift_prompt_train(self): from swift import PromptConfig model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-prompt' diff --git a/tests/trainers/test_general_image_classification_trainer.py b/tests/trainers/test_general_image_classification_trainer.py index 6aba96dc..1bd071fc 100644 --- a/tests/trainers/test_general_image_classification_trainer.py +++ b/tests/trainers/test_general_image_classification_trainer.py @@ -12,6 +12,7 @@ from modelscope.msdatasets import MsDataset from modelscope.trainers import build_trainer from modelscope.utils.config import Config, ConfigDict from modelscope.utils.constant import DownloadMode, ModelFile +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -45,7 +46,7 @@ class TestGeneralImageClassificationTestTrainer(unittest.TestCase): shutil.rmtree(self.tmp_dir) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(exists('mmcv'), 'Skip because mmcv is not installed.') def test_nextvit_dailylife_train(self): model_id = 'damo/cv_nextvit-small_image-classification_Dailylife-labels' @@ -76,7 +77,7 @@ class TestGeneralImageClassificationTestTrainer(unittest.TestCase): for i in range(self.max_epochs): self.assertIn(f'epoch_{i+1}.pth', results_files) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(exists('mmcv'), 'Skip because mmcv is not installed.') def test_nextvit_dailylife_eval(self): model_id = 'damo/cv_nextvit-small_image-classification_Dailylife-labels' @@ -91,7 +92,7 @@ class TestGeneralImageClassificationTestTrainer(unittest.TestCase): result = trainer.evaluate() print(result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(exists('mmcv'), 'Skip because mmcv is not installed.') def test_convnext_garbage_train(self): model_id = 'damo/cv_convnext-base_image-classification_garbage' @@ -122,7 +123,7 @@ class TestGeneralImageClassificationTestTrainer(unittest.TestCase): for i in range(self.max_epochs): self.assertIn(f'epoch_{i+1}.pth', results_files) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(exists('mmcv'), 'Skip because mmcv is not installed.') def test_convnext_garbage_eval(self): model_id = 'damo/cv_convnext-base_image-classification_garbage' @@ -137,7 +138,7 @@ class TestGeneralImageClassificationTestTrainer(unittest.TestCase): result = trainer.evaluate() print(result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(exists('mmcv'), 'Skip because mmcv is not installed.') def test_beitv2_train_eval(self): model_id = 'damo/cv_beitv2-base_image-classification_patch16_224_pt1k_ft22k_in1k' diff --git a/tests/trainers/test_image_color_enhance_trainer.py b/tests/trainers/test_image_color_enhance_trainer.py index 34d84cd2..45259cac 100644 --- a/tests/trainers/test_image_color_enhance_trainer.py +++ b/tests/trainers/test_image_color_enhance_trainer.py @@ -4,14 +4,13 @@ import os.path as osp import shutil import tempfile import unittest -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, List, Union import cv2 import torch from torch.utils import data as data from modelscope.hub.snapshot_download import snapshot_download -from modelscope.models.cv.image_color_enhance import ImageColorEnhance from modelscope.trainers import build_trainer from modelscope.utils.constant import ModelFile from modelscope.utils.test_utils import test_level @@ -69,7 +68,7 @@ class TestImageColorEnhanceTrainer(unittest.TestCase): shutil.rmtree(self.tmp_dir, ignore_errors=True) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_trainer(self): kwargs = dict( model=self.model_id, @@ -86,6 +85,7 @@ class TestImageColorEnhanceTrainer(unittest.TestCase): @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_trainer_with_model_and_args(self): + from modelscope.models.cv.image_color_enhance import ImageColorEnhance cache_path = snapshot_download(self.model_id) model = ImageColorEnhance.from_pretrained(cache_path) kwargs = dict( diff --git a/tests/trainers/test_image_instance_segmentation_trainer.py b/tests/trainers/test_image_instance_segmentation_trainer.py index 923eca2c..ac680992 100644 --- a/tests/trainers/test_image_instance_segmentation_trainer.py +++ b/tests/trainers/test_image_instance_segmentation_trainer.py @@ -3,13 +3,10 @@ import os import shutil import tempfile import unittest -import zipfile from functools import partial from modelscope.hub.snapshot_download import snapshot_download from modelscope.metainfo import Trainers -from modelscope.models.cv.image_instance_segmentation import \ - CascadeMaskRCNNSwinModel from modelscope.msdatasets import MsDataset from modelscope.trainers import build_trainer from modelscope.utils.config import Config, ConfigDict @@ -76,7 +73,7 @@ class TestImageInstanceSegmentationTrainer(unittest.TestCase): shutil.rmtree(self.tmp_dir) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_trainer(self): kwargs = dict( model=self.model_id, @@ -95,6 +92,8 @@ class TestImageInstanceSegmentationTrainer(unittest.TestCase): @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_trainer_with_model_and_args(self): + from modelscope.models.cv.image_instance_segmentation import \ + CascadeMaskRCNNSwinModel tmp_dir = tempfile.TemporaryDirectory().name if not os.path.exists(tmp_dir): os.makedirs(tmp_dir) diff --git a/tests/trainers/test_movie_scene_segmentation_trainer.py b/tests/trainers/test_movie_scene_segmentation_trainer.py index d598a63f..93826e58 100644 --- a/tests/trainers/test_movie_scene_segmentation_trainer.py +++ b/tests/trainers/test_movie_scene_segmentation_trainer.py @@ -3,12 +3,9 @@ import os import shutil import tempfile import unittest -import zipfile from modelscope.hub.snapshot_download import snapshot_download from modelscope.metainfo import Trainers -from modelscope.models.cv.movie_scene_segmentation import \ - MovieSceneSegmentationModel from modelscope.msdatasets import MsDataset from modelscope.trainers import build_trainer from modelscope.utils.config import Config, ConfigDict @@ -86,7 +83,8 @@ class TestImageInstanceSegmentationTrainer(unittest.TestCase): tmp_dir = tempfile.TemporaryDirectory().name if not os.path.exists(tmp_dir): os.makedirs(tmp_dir) - + from modelscope.models.cv.movie_scene_segmentation import \ + MovieSceneSegmentationModel cache_path = snapshot_download(self.model_id) model = MovieSceneSegmentationModel.from_pretrained(cache_path) kwargs = dict( diff --git a/tests/trainers/test_nerf_recon_acc_trainer.py b/tests/trainers/test_nerf_recon_acc_trainer.py index cc42ca1f..0ba14c06 100644 --- a/tests/trainers/test_nerf_recon_acc_trainer.py +++ b/tests/trainers/test_nerf_recon_acc_trainer.py @@ -4,7 +4,6 @@ import shutil import unittest from modelscope.msdatasets import MsDataset -from modelscope.trainers.cv import NeRFReconAccTrainer from modelscope.utils.constant import DownloadMode from modelscope.utils.test_utils import test_level @@ -25,7 +24,7 @@ class TestNeRFReconAccTrainer(unittest.TestCase): split='train', download_mode=DownloadMode.FORCE_REDOWNLOAD ).config_kwargs['split_config']['train'] - + from modelscope.trainers.cv import NeRFReconAccTrainer trainer = NeRFReconAccTrainer( model=model_id, data_type='blender', diff --git a/tests/trainers/test_ocr_detection_db_trainer.py b/tests/trainers/test_ocr_detection_db_trainer.py index 136ae410..f91a0b49 100644 --- a/tests/trainers/test_ocr_detection_db_trainer.py +++ b/tests/trainers/test_ocr_detection_db_trainer.py @@ -13,6 +13,7 @@ from modelscope.pipelines import pipeline from modelscope.trainers import build_trainer from modelscope.utils.config import Config from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import DistributedTestCase, test_level @@ -40,7 +41,9 @@ class TestOCRDetectionDBTrainerSingleGPU(unittest.TestCase): shutil.rmtree(self.saved_dir) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer_finetune_singleGPU(self): kwargs = dict( diff --git a/tests/trainers/test_ofa_mmspeech_trainer.py b/tests/trainers/test_ofa_mmspeech_trainer.py index 6ab4ac76..8af4d9f7 100644 --- a/tests/trainers/test_ofa_mmspeech_trainer.py +++ b/tests/trainers/test_ofa_mmspeech_trainer.py @@ -9,6 +9,7 @@ from modelscope.metainfo import Trainers from modelscope.msdatasets import MsDataset from modelscope.trainers import build_trainer from modelscope.utils.constant import DownloadMode, ModelFile +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -76,7 +77,8 @@ class TestMMSpeechTrainer(unittest.TestCase): shutil.rmtree(self.WORKSPACE, ignore_errors=True) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('fairseq'), 'Skip because fairseq is not installed.') def test_trainer_std(self): os.makedirs(self.WORKSPACE, exist_ok=True) config_file = os.path.join(self.WORKSPACE, ModelFile.CONFIGURATION) diff --git a/tests/trainers/test_referring_video_object_segmentation_trainer.py b/tests/trainers/test_referring_video_object_segmentation_trainer.py index 53ab5236..c841c1ca 100644 --- a/tests/trainers/test_referring_video_object_segmentation_trainer.py +++ b/tests/trainers/test_referring_video_object_segmentation_trainer.py @@ -1,18 +1,15 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import os import shutil -import tempfile import unittest -import zipfile from modelscope.hub.snapshot_download import snapshot_download from modelscope.metainfo import Trainers -from modelscope.models.cv.referring_video_object_segmentation import \ - ReferringVideoObjectSegmentation from modelscope.msdatasets import MsDataset from modelscope.trainers import build_trainer from modelscope.utils.config import Config, ConfigDict from modelscope.utils.constant import ModelFile +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -64,7 +61,9 @@ class TestImageInstanceSegmentationTrainer(unittest.TestCase): shutil.rmtree('./work_dir') super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer(self): kwargs = dict( model=self.model_id, @@ -81,7 +80,8 @@ class TestImageInstanceSegmentationTrainer(unittest.TestCase): @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_trainer_with_model_and_args(self): - + from modelscope.models.cv.referring_video_object_segmentation import \ + ReferringVideoObjectSegmentation cache_path = snapshot_download(self.model_id) model = ReferringVideoObjectSegmentation.from_pretrained(cache_path) kwargs = dict( diff --git a/tests/trainers/test_table_question_answering_trainer.py b/tests/trainers/test_table_question_answering_trainer.py index b168dcfc..921109c8 100644 --- a/tests/trainers/test_table_question_answering_trainer.py +++ b/tests/trainers/test_table_question_answering_trainer.py @@ -13,7 +13,7 @@ from modelscope.utils.test_utils import test_level class TableQuestionAnsweringTest(unittest.TestCase): - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_trainer_with_model_name(self): # load data input_dataset = MsDataset.load( diff --git a/tests/trainers/test_tinynas_damoyolo_trainer.py b/tests/trainers/test_tinynas_damoyolo_trainer.py index 52311d83..5b48f8d2 100644 --- a/tests/trainers/test_tinynas_damoyolo_trainer.py +++ b/tests/trainers/test_tinynas_damoyolo_trainer.py @@ -8,6 +8,7 @@ import unittest from modelscope.hub.snapshot_download import snapshot_download from modelscope.metainfo import Trainers from modelscope.trainers import build_trainer +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -29,7 +30,9 @@ class TestTinynasDamoyoloTrainerSingleGPU(unittest.TestCase): super().tearDown() shutil.rmtree('./workdirs', ignore_errors=True) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer_from_scratch_singleGPU(self): kwargs = dict( cfg_file=os.path.join(self.cache_path, 'configuration.json'), @@ -57,7 +60,9 @@ class TestTinynasDamoyoloTrainerSingleGPU(unittest.TestCase): checkpoint_path=os.path.join('./workdirs/damoyolo_s', 'epoch_3_ckpt.pth')) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer_from_scratch_singleGPU_model_id(self): kwargs = dict( model=self.model_id, @@ -110,7 +115,9 @@ class TestTinynasDamoyoloTrainerSingleGPU(unittest.TestCase): name=Trainers.tinynas_damoyolo, default_args=kwargs) trainer.train() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer_finetune_singleGPU(self): kwargs = dict( cfg_file=os.path.join(self.cache_path, 'configuration.json'), diff --git a/tests/trainers/test_trainer.py b/tests/trainers/test_trainer.py index 2cf4b2e9..28841ab9 100644 --- a/tests/trainers/test_trainer.py +++ b/tests/trainers/test_trainer.py @@ -23,6 +23,7 @@ from modelscope.trainers.builder import TRAINERS from modelscope.trainers.trainer import EpochBasedTrainer from modelscope.utils.constant import LogKeys, ModeKeys, ModelFile, Tasks from modelscope.utils.hub import read_config +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import create_dummy_test_dataset, test_level @@ -84,7 +85,9 @@ class TrainerTest(unittest.TestCase): super().tearDown() shutil.rmtree(self.tmp_dir) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_train_0(self): json_cfg = { 'task': Tasks.image_classification, @@ -167,7 +170,9 @@ class TrainerTest(unittest.TestCase): self.assertIn('tensorboard_output', results_files) self.assertTrue(len(glob.glob(f'{self.tmp_dir}/*/*events*')) > 0) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_train_visualization(self): json_cfg = { 'task': Tasks.image_classification, @@ -247,7 +252,9 @@ class TrainerTest(unittest.TestCase): self.assertIn(f'{LogKeys.EPOCH}_3.pth', results_files) self.assertTrue(len(glob.glob(f'{self.tmp_dir}/*/*events*')) > 0) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_train_1(self): json_cfg = { 'task': Tasks.image_classification, @@ -312,7 +319,9 @@ class TrainerTest(unittest.TestCase): self.assertIn(f'{LogKeys.EPOCH}_3.pth', results_files) self.assertTrue(len(glob.glob(f'{self.tmp_dir}/*/*events*')) > 0) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_train_with_default_config(self): json_cfg = { 'task': Tasks.image_classification, @@ -431,7 +440,9 @@ class TrainerTest(unittest.TestCase): for i in [2, 5, 8]: self.assertIn(MetricKeys.ACCURACY, lines[i]) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_train_with_iters_per_epoch(self): json_cfg = { 'task': Tasks.image_classification, @@ -550,7 +561,9 @@ class TrainerTest(unittest.TestCase): for i in [2, 5, 8]: self.assertIn(MetricKeys.ACCURACY, lines[i]) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_train_with_old_and_new_cfg(self): old_cfg = { 'task': Tasks.image_classification, @@ -693,7 +706,9 @@ class TrainerTest(unittest.TestCase): class DummyTrainerTest(unittest.TestCase): - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_dummy(self): default_args = dict(cfg_file='configs/examples/train.json') trainer = build_trainer('dummy', default_args) diff --git a/tests/trainers/test_trainer_with_nlp.py b/tests/trainers/test_trainer_with_nlp.py index a736d4fa..90ec8b5e 100644 --- a/tests/trainers/test_trainer_with_nlp.py +++ b/tests/trainers/test_trainer_with_nlp.py @@ -21,6 +21,7 @@ from modelscope.trainers import EpochBasedTrainer, build_trainer from modelscope.utils.config import Config from modelscope.utils.constant import ModelFile, Tasks from modelscope.utils.hub import read_config +from modelscope.utils.import_utils import exists from modelscope.utils.test_utils import test_level @@ -42,7 +43,9 @@ class TestTrainerWithNlp(unittest.TestCase): shutil.rmtree(self.tmp_dir) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer(self): model_id = 'damo/nlp_structbert_sentence-similarity_chinese-tiny' kwargs = dict( @@ -79,7 +82,9 @@ class TestTrainerWithNlp(unittest.TestCase): output_dir = os.path.join(self.tmp_dir, ModelFile.TRAIN_OUTPUT_DIR) pipeline_sentence_similarity(output_dir) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer_callback(self): model_id = 'damo/nlp_structbert_sentence-similarity_chinese-tiny' @@ -101,9 +106,8 @@ class TestTrainerWithNlp(unittest.TestCase): self.assertEqual(trainer.iter, 3) - @unittest.skipIf( - version.parse(torch.__version__) < version.parse('2.0.0.dev'), - 'skip test when torch version < 2.0') + @unittest.skipUnless( + exists('torch<2.4'), 'Skip test because torch version is too high.') def test_trainer_compile(self): model_id = 'damo/nlp_structbert_sentence-similarity_chinese-tiny' @@ -406,7 +410,9 @@ class TestTrainerWithNlp(unittest.TestCase): trainer, 'trainer_continue_train', level='strict'): trainer.train(os.path.join(self.tmp_dir, 'iter_3')) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer_with_new_style_configuration(self): tmp_dir = tempfile.TemporaryDirectory().name if not os.path.exists(tmp_dir): @@ -481,7 +487,9 @@ class TestTrainerWithNlp(unittest.TestCase): cache_path + '/pytorch_model.bin', saving_fn=saving_fn)) self.assertTrue(os.path.isfile(f'{tmp_dir}/predicts.txt')) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer_with_custom_sampler(self): tmp_dir = tempfile.TemporaryDirectory().name if not os.path.exists(tmp_dir): @@ -509,7 +517,9 @@ class TestTrainerWithNlp(unittest.TestCase): type(trainer.train_dataloader.sampler) == CustomSampler) self.assertTrue(type(trainer.eval_dataloader.sampler) == CustomSampler) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer_with_prediction(self): tmp_dir = tempfile.TemporaryDirectory().name if not os.path.exists(tmp_dir): @@ -551,7 +561,9 @@ class TestTrainerWithNlp(unittest.TestCase): checkpoint_path=cache_path + '/pytorch_model.bin') self.assertTrue(os.path.isfile(f'{tmp_dir}/predicts.txt')) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer_with_prediction_msdataset(self): tmp_dir = tempfile.TemporaryDirectory().name if not os.path.exists(tmp_dir): @@ -608,7 +620,9 @@ class TestTrainerWithNlp(unittest.TestCase): for i in range(2): self.assertIn(f'epoch_{i + 1}.pth', results_files) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer_with_hook_register(self): model_id = 'damo/nlp_structbert_sentence-similarity_chinese-tiny' diff --git a/tests/trainers/test_translation_trainer.py b/tests/trainers/test_translation_trainer.py index ef0c6e76..47ded16c 100644 --- a/tests/trainers/test_translation_trainer.py +++ b/tests/trainers/test_translation_trainer.py @@ -1,26 +1,31 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import unittest -from modelscope.trainers.nlp import CsanmtTranslationTrainer -from modelscope.utils.test_utils import test_level +from modelscope.utils.import_utils import exists class TranslationTest(unittest.TestCase): - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('tensorflow'), 'Skip test because tensorflow is not installed.') def test_run_with_model_name_for_en2zh(self): + from modelscope.trainers.nlp import CsanmtTranslationTrainer model_id = 'damo/nlp_csanmt_translation_en2zh' trainer = CsanmtTranslationTrainer(model=model_id) trainer.train() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('tensorflow'), 'Skip test because tensorflow is not installed.') def test_run_with_model_name_for_en2fr(self): + from modelscope.trainers.nlp import CsanmtTranslationTrainer model_id = 'damo/nlp_csanmt_translation_en2fr' trainer = CsanmtTranslationTrainer(model=model_id) trainer.train() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('tensorflow'), 'Skip test because tensorflow is not installed.') def test_run_with_model_name_for_en2es(self): + from modelscope.trainers.nlp import CsanmtTranslationTrainer model_id = 'damo/nlp_csanmt_translation_en2es' trainer = CsanmtTranslationTrainer(model=model_id) trainer.train() diff --git a/tests/trainers/test_video_summarization_trainer.py b/tests/trainers/test_video_summarization_trainer.py index 35eee2bc..bc236ad3 100644 --- a/tests/trainers/test_video_summarization_trainer.py +++ b/tests/trainers/test_video_summarization_trainer.py @@ -11,6 +11,7 @@ from modelscope.msdatasets.dataset_cls.custom_datasets import \ from modelscope.trainers import build_trainer from modelscope.utils.config import Config from modelscope.utils.constant import ModelFile +from modelscope.utils.import_utils import exists from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import test_level @@ -41,7 +42,9 @@ class VideoSummarizationTrainerTest(unittest.TestCase): shutil.rmtree(self.tmp_dir, ignore_errors=True) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + exists('transformers<5.0'), + 'Skip test because transformers version is too high.') def test_trainer(self): kwargs = dict( model=self.model_id, diff --git a/tests/utils/test_compatibility.py b/tests/utils/test_compatibility.py index f5222261..8b38bf00 100644 --- a/tests/utils/test_compatibility.py +++ b/tests/utils/test_compatibility.py @@ -2,6 +2,8 @@ import unittest +from modelscope.utils.import_utils import exists + class CompatibilityTest(unittest.TestCase): @@ -11,6 +13,8 @@ class CompatibilityTest(unittest.TestCase): def tearDown(self): super().tearDown() + @unittest.skipUnless( + exists('xtcocotools'), 'Skip because xtcocotools is not installed.') def test_xtcocotools(self): from xtcocotools.coco import COCO diff --git a/tests/utils/test_device.py b/tests/utils/test_device.py index 0d334fda..93799f48 100644 --- a/tests/utils/test_device.py +++ b/tests/utils/test_device.py @@ -12,9 +12,6 @@ from modelscope.utils.constant import Frameworks from modelscope.utils.device import (create_device, device_placement, verify_device) -# import tensorflow must be imported after torch is imported when using tf1.15 -import tensorflow as tf # isort:skip - class DeviceTest(unittest.TestCase): @@ -89,6 +86,8 @@ class DeviceTest(unittest.TestCase): @unittest.skip('skip this test to avoid debug logging.') def test_device_placement_tf_gpu(self): + # import tensorflow must be imported after torch is imported when using tf1.15 + import tensorflow as tf # isort:skip tf.debugging.set_log_device_placement(True) with device_placement(Frameworks.tf, 'gpu:0'): a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]) diff --git a/tests/utils/test_hf_util.py b/tests/utils/test_hf_util.py index 900eb867..57695068 100644 --- a/tests/utils/test_hf_util.py +++ b/tests/utils/test_hf_util.py @@ -23,7 +23,7 @@ class HFUtilTest(unittest.TestCase): def setUp(self): logger.info('SetUp') self.api = HubApi() - self.api.login(TEST_ACCESS_TOKEN1) + response, _ = self.api.login(TEST_ACCESS_TOKEN1) self.user = TEST_MODEL_ORG print(self.user) self.create_model_name = '%s/%s_%s' % (self.user, 'test_model_upload', @@ -80,6 +80,7 @@ class HFUtilTest(unittest.TestCase): 'baichuan-inc/baichuan-7B', trust_remote_code=True) self.assertTrue(model is not None) + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_auto_config(self): from modelscope import AutoConfig, GenerationConfig config = AutoConfig.from_pretrained( @@ -108,33 +109,19 @@ class HFUtilTest(unittest.TestCase): def test_transformer_patch(self): with patch_context(): from transformers import AutoTokenizer, AutoModelForCausalLM - tokenizer = AutoTokenizer.from_pretrained( - 'iic/nlp_structbert_sentiment-classification_chinese-tiny') + tokenizer = AutoTokenizer.from_pretrained('Qwen/Qwen2.5-0.5B') self.assertIsNotNone(tokenizer) - model = AutoModelForCausalLM.from_pretrained( - 'iic/nlp_structbert_sentiment-classification_chinese-tiny') + model = AutoModelForCausalLM.from_pretrained('Qwen/Qwen2.5-0.5B') self.assertIsNotNone(model) def test_patch_model(self): from modelscope.utils.hf_util.patcher import patch_context with patch_context(): from transformers import AutoModel - model = AutoModel.from_pretrained( - 'iic/nlp_structbert_sentiment-classification_chinese-tiny') + model = AutoModel.from_pretrained('Qwen/Qwen2.5-0.5B') self.assertTrue(model is not None) try: - model = AutoModel.from_pretrained( - 'iic/nlp_structbert_sentiment-classification_chinese-tiny') - except Exception: - pass - else: - self.assertTrue(False) - - def test_patch_config_bert(self): - from transformers import BertConfig - try: - BertConfig.from_pretrained( - 'iic/nlp_structbert_sentiment-classification_chinese-tiny') + model = AutoModel.from_pretrained('Qwen/Qwen2.5-0.5B') except Exception: pass else: @@ -143,22 +130,18 @@ class HFUtilTest(unittest.TestCase): def test_patch_config(self): with patch_context(): from transformers import AutoConfig - config = AutoConfig.from_pretrained( - 'iic/nlp_structbert_sentiment-classification_chinese-tiny') - self.assertTrue(getattr(config, 'base_model_prefix') == 'encoder') + config = AutoConfig.from_pretrained('Qwen/Qwen2.5-0.5B') + self.assertTrue(config is not None) try: - config = AutoConfig.from_pretrained( - 'iic/nlp_structbert_sentiment-classification_chinese-tiny') - self.assertTrue( - getattr(config, 'base_model_prefix', None) != 'encoder') + AutoConfig.from_pretrained('Qwen/Qwen2.5-0.5B') + self.assertTrue(False) except: # noqa pass # Test patch again with patch_context(): from transformers import AutoConfig - config = AutoConfig.from_pretrained( - 'iic/nlp_structbert_sentiment-classification_chinese-tiny') + config = AutoConfig.from_pretrained('Qwen/Qwen2.5-0.5B') self.assertTrue(config is not None) @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') diff --git a/tests/utils/test_megatron.py b/tests/utils/test_megatron.py index 0af30cd4..f24defb2 100644 --- a/tests/utils/test_megatron.py +++ b/tests/utils/test_megatron.py @@ -25,7 +25,7 @@ class MegatronTest(DistributedTestCase): shutil.rmtree(self.tmp_dir) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skip def test_init_megatron_util(self): dummy_megatron_cfg = { 'tensor_model_parallel_size': 1, @@ -37,8 +37,7 @@ class MegatronTest(DistributedTestCase): init_megatron_util(dummy_megatron_cfg) self.assertTrue(is_megatron_initialized()) - @unittest.skipIf(not torch.cuda.is_available() - or torch.cuda.device_count() <= 1, 'distributed unittest') + @unittest.skip def test_convert_megatron_checkpoint(self): cache_path = snapshot_download('damo/nlp_gpt3_text-generation_1.3B') splited_dir = os.path.join(self.tmp_dir, 'splited') diff --git a/tests/utils/test_sentence_embedding_utils.py b/tests/utils/test_sentence_embedding_utils.py index 5e4cf76e..16e214f2 100644 --- a/tests/utils/test_sentence_embedding_utils.py +++ b/tests/utils/test_sentence_embedding_utils.py @@ -2,6 +2,7 @@ import subprocess import sys import unittest +from modelscope.models.cv.anydoor.ldm.util import exists from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.test_utils import test_level @@ -24,6 +25,9 @@ class SentenceEmbeddingPipelineTest(unittest.TestCase): 'is responsible for the movement of planets around the sun.', ] + @unittest.skipUnless( + exists('sentence_transformers'), + 'Skip because sentence_transformers is not installed.') def test_ori_pipeline(self): ppl = pipeline( Tasks.sentence_embedding,