mirror of
https://github.com/modelscope/modelscope.git
synced 2026-09-01 19:49:03 +02:00
[to #47671666]fix: diff based ci optimize
Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/11574741 * [to #47671666]fix: diff based ci optimize
This commit is contained in:
mulin.lyh
@@ -1,5 +1,5 @@
|
||||
# Copyright (c) Alibaba, Inc. and its affiliates.
|
||||
from modelscope.utils.constant import Fields
|
||||
from modelscope.utils.constant import Fields, Tasks
|
||||
|
||||
|
||||
class Models(object):
|
||||
@@ -400,6 +400,272 @@ class Pipelines(object):
|
||||
protein_structure = 'unifold-protein-structure'
|
||||
|
||||
|
||||
DEFAULT_MODEL_FOR_PIPELINE = {
|
||||
# TaskName: (pipeline_module_name, model_repo)
|
||||
Tasks.sentence_embedding:
|
||||
(Pipelines.sentence_embedding,
|
||||
'damo/nlp_corom_sentence-embedding_english-base'),
|
||||
Tasks.text_ranking: (Pipelines.text_ranking,
|
||||
'damo/nlp_corom_passage-ranking_english-base'),
|
||||
Tasks.text_ranking: (Pipelines.mgeo_ranking,
|
||||
'damo/mgeo_address_ranking_chinese_base'),
|
||||
Tasks.word_segmentation:
|
||||
(Pipelines.word_segmentation,
|
||||
'damo/nlp_structbert_word-segmentation_chinese-base'),
|
||||
Tasks.part_of_speech: (Pipelines.part_of_speech,
|
||||
'damo/nlp_structbert_part-of-speech_chinese-base'),
|
||||
Tasks.token_classification:
|
||||
(Pipelines.part_of_speech,
|
||||
'damo/nlp_structbert_part-of-speech_chinese-base'),
|
||||
Tasks.named_entity_recognition:
|
||||
(Pipelines.named_entity_recognition,
|
||||
'damo/nlp_raner_named-entity-recognition_chinese-base-news'),
|
||||
Tasks.relation_extraction:
|
||||
(Pipelines.relation_extraction,
|
||||
'damo/nlp_bert_relation-extraction_chinese-base'),
|
||||
Tasks.information_extraction:
|
||||
(Pipelines.relation_extraction,
|
||||
'damo/nlp_bert_relation-extraction_chinese-base'),
|
||||
Tasks.sentence_similarity:
|
||||
(Pipelines.sentence_similarity,
|
||||
'damo/nlp_structbert_sentence-similarity_chinese-base'),
|
||||
Tasks.translation: (Pipelines.csanmt_translation,
|
||||
'damo/nlp_csanmt_translation_zh2en'),
|
||||
Tasks.nli: (Pipelines.nli, 'damo/nlp_structbert_nli_chinese-base'),
|
||||
Tasks.sentiment_classification:
|
||||
(Pipelines.sentiment_classification,
|
||||
'damo/nlp_structbert_sentiment-classification_chinese-base'
|
||||
), # TODO: revise back after passing the pr
|
||||
Tasks.portrait_matting: (Pipelines.portrait_matting,
|
||||
'damo/cv_unet_image-matting'),
|
||||
Tasks.human_detection: (Pipelines.human_detection,
|
||||
'damo/cv_resnet18_human-detection'),
|
||||
Tasks.image_object_detection: (Pipelines.object_detection,
|
||||
'damo/cv_vit_object-detection_coco'),
|
||||
Tasks.image_denoising: (Pipelines.image_denoise,
|
||||
'damo/cv_nafnet_image-denoise_sidd'),
|
||||
Tasks.image_deblurring: (Pipelines.image_deblur,
|
||||
'damo/cv_nafnet_image-deblur_gopro'),
|
||||
Tasks.video_stabilization: (Pipelines.video_stabilization,
|
||||
'damo/cv_dut-raft_video-stabilization_base'),
|
||||
Tasks.video_super_resolution:
|
||||
(Pipelines.video_super_resolution,
|
||||
'damo/cv_realbasicvsr_video-super-resolution_videolq'),
|
||||
Tasks.text_classification:
|
||||
(Pipelines.sentiment_classification,
|
||||
'damo/nlp_structbert_sentiment-classification_chinese-base'),
|
||||
Tasks.text_generation: (Pipelines.text_generation,
|
||||
'damo/nlp_palm2.0_text-generation_chinese-base'),
|
||||
Tasks.zero_shot_classification:
|
||||
(Pipelines.zero_shot_classification,
|
||||
'damo/nlp_structbert_zero-shot-classification_chinese-base'),
|
||||
Tasks.task_oriented_conversation: (Pipelines.dialog_modeling,
|
||||
'damo/nlp_space_dialog-modeling'),
|
||||
Tasks.dialog_state_tracking: (Pipelines.dialog_state_tracking,
|
||||
'damo/nlp_space_dialog-state-tracking'),
|
||||
Tasks.table_question_answering:
|
||||
(Pipelines.table_question_answering_pipeline,
|
||||
'damo/nlp-convai-text2sql-pretrain-cn'),
|
||||
Tasks.text_error_correction:
|
||||
(Pipelines.text_error_correction,
|
||||
'damo/nlp_bart_text-error-correction_chinese'),
|
||||
Tasks.image_captioning: (Pipelines.image_captioning,
|
||||
'damo/ofa_image-caption_coco_large_en'),
|
||||
Tasks.video_captioning:
|
||||
(Pipelines.video_captioning,
|
||||
'damo/multi-modal_hitea_video-captioning_base_en'),
|
||||
Tasks.image_portrait_stylization:
|
||||
(Pipelines.person_image_cartoon,
|
||||
'damo/cv_unet_person-image-cartoon_compound-models'),
|
||||
Tasks.ocr_detection: (Pipelines.ocr_detection,
|
||||
'damo/cv_resnet18_ocr-detection-line-level_damo'),
|
||||
Tasks.table_recognition:
|
||||
(Pipelines.table_recognition,
|
||||
'damo/cv_dla34_table-structure-recognition_cycle-centernet'),
|
||||
Tasks.document_vl_embedding:
|
||||
(Pipelines.document_vl_embedding,
|
||||
'damo/multi-modal_convnext-roberta-base_vldoc-embedding'),
|
||||
Tasks.license_plate_detection:
|
||||
(Pipelines.license_plate_detection,
|
||||
'damo/cv_resnet18_license-plate-detection_damo'),
|
||||
Tasks.fill_mask: (Pipelines.fill_mask, 'damo/nlp_veco_fill-mask-large'),
|
||||
Tasks.feature_extraction: (Pipelines.feature_extraction,
|
||||
'damo/pert_feature-extraction_base-test'),
|
||||
Tasks.action_recognition: (Pipelines.action_recognition,
|
||||
'damo/cv_TAdaConv_action-recognition'),
|
||||
Tasks.action_detection: (Pipelines.action_detection,
|
||||
'damo/cv_ResNetC3D_action-detection_detection2d'),
|
||||
Tasks.live_category: (Pipelines.live_category,
|
||||
'damo/cv_resnet50_live-category'),
|
||||
Tasks.video_category: (Pipelines.video_category,
|
||||
'damo/cv_resnet50_video-category'),
|
||||
Tasks.multi_modal_embedding: (Pipelines.multi_modal_embedding,
|
||||
'damo/multi-modal_clip-vit-base-patch16_zh'),
|
||||
Tasks.generative_multi_modal_embedding:
|
||||
(Pipelines.generative_multi_modal_embedding,
|
||||
'damo/multi-modal_gemm-vit-large-patch14_generative-multi-modal-embedding'
|
||||
),
|
||||
Tasks.multi_modal_similarity:
|
||||
(Pipelines.multi_modal_similarity,
|
||||
'damo/multi-modal_team-vit-large-patch14_multi-modal-similarity'),
|
||||
Tasks.visual_question_answering:
|
||||
(Pipelines.visual_question_answering,
|
||||
'damo/mplug_visual-question-answering_coco_large_en'),
|
||||
Tasks.video_question_answering:
|
||||
(Pipelines.video_question_answering,
|
||||
'damo/multi-modal_hitea_video-question-answering_base_en'),
|
||||
Tasks.video_embedding: (Pipelines.cmdssl_video_embedding,
|
||||
'damo/cv_r2p1d_video_embedding'),
|
||||
Tasks.text_to_image_synthesis:
|
||||
(Pipelines.text_to_image_synthesis,
|
||||
'damo/cv_diffusion_text-to-image-synthesis_tiny'),
|
||||
Tasks.body_2d_keypoints: (Pipelines.body_2d_keypoints,
|
||||
'damo/cv_hrnetv2w32_body-2d-keypoints_image'),
|
||||
Tasks.body_3d_keypoints: (Pipelines.body_3d_keypoints,
|
||||
'damo/cv_canonical_body-3d-keypoints_video'),
|
||||
Tasks.hand_2d_keypoints:
|
||||
(Pipelines.hand_2d_keypoints,
|
||||
'damo/cv_hrnetw18_hand-pose-keypoints_coco-wholebody'),
|
||||
Tasks.card_detection: (Pipelines.card_detection,
|
||||
'damo/cv_resnet_carddetection_scrfd34gkps'),
|
||||
Tasks.face_detection:
|
||||
(Pipelines.mog_face_detection,
|
||||
'damo/cv_resnet101_face-detection_cvpr22papermogface'),
|
||||
Tasks.face_liveness: (Pipelines.face_liveness_ir,
|
||||
'damo/cv_manual_face-liveness_flir'),
|
||||
Tasks.face_recognition: (Pipelines.face_recognition,
|
||||
'damo/cv_ir101_facerecognition_cfglint'),
|
||||
Tasks.facial_expression_recognition:
|
||||
(Pipelines.facial_expression_recognition,
|
||||
'damo/cv_vgg19_facial-expression-recognition_fer'),
|
||||
Tasks.face_attribute_recognition:
|
||||
(Pipelines.face_attribute_recognition,
|
||||
'damo/cv_resnet34_face-attribute-recognition_fairface'),
|
||||
Tasks.face_2d_keypoints: (Pipelines.face_2d_keypoints,
|
||||
'damo/cv_mobilenet_face-2d-keypoints_alignment'),
|
||||
Tasks.video_multi_modal_embedding:
|
||||
(Pipelines.video_multi_modal_embedding,
|
||||
'damo/multi_modal_clip_vtretrival_msrvtt_53'),
|
||||
Tasks.image_color_enhancement:
|
||||
(Pipelines.image_color_enhance,
|
||||
'damo/cv_csrnet_image-color-enhance-models'),
|
||||
Tasks.virtual_try_on: (Pipelines.virtual_try_on,
|
||||
'damo/cv_daflow_virtual-try-on_base'),
|
||||
Tasks.image_colorization: (Pipelines.ddcolor_image_colorization,
|
||||
'damo/cv_ddcolor_image-colorization'),
|
||||
Tasks.video_colorization: (Pipelines.video_colorization,
|
||||
'damo/cv_unet_video-colorization'),
|
||||
Tasks.image_segmentation:
|
||||
(Pipelines.image_instance_segmentation,
|
||||
'damo/cv_swin-b_image-instance-segmentation_coco'),
|
||||
Tasks.image_depth_estimation:
|
||||
(Pipelines.image_depth_estimation,
|
||||
'damo/cv_newcrfs_image-depth-estimation_indoor'),
|
||||
Tasks.indoor_layout_estimation:
|
||||
(Pipelines.indoor_layout_estimation,
|
||||
'damo/cv_panovit_indoor-layout-estimation'),
|
||||
Tasks.video_depth_estimation:
|
||||
(Pipelines.video_depth_estimation,
|
||||
'damo/cv_dro-resnet18_video-depth-estimation_indoor'),
|
||||
Tasks.panorama_depth_estimation:
|
||||
(Pipelines.panorama_depth_estimation,
|
||||
'damo/cv_unifuse_panorama-depth-estimation'),
|
||||
Tasks.image_style_transfer: (Pipelines.image_style_transfer,
|
||||
'damo/cv_aams_style-transfer_damo'),
|
||||
Tasks.face_image_generation: (Pipelines.face_image_generation,
|
||||
'damo/cv_gan_face-image-generation'),
|
||||
Tasks.image_super_resolution: (Pipelines.image_super_resolution,
|
||||
'damo/cv_rrdb_image-super-resolution'),
|
||||
Tasks.image_portrait_enhancement:
|
||||
(Pipelines.image_portrait_enhancement,
|
||||
'damo/cv_gpen_image-portrait-enhancement'),
|
||||
Tasks.product_retrieval_embedding:
|
||||
(Pipelines.product_retrieval_embedding,
|
||||
'damo/cv_resnet50_product-bag-embedding-models'),
|
||||
Tasks.image_to_image_generation:
|
||||
(Pipelines.image_to_image_generation,
|
||||
'damo/cv_latent_diffusion_image2image_generate'),
|
||||
Tasks.image_classification:
|
||||
(Pipelines.daily_image_classification,
|
||||
'damo/cv_vit-base_image-classification_Dailylife-labels'),
|
||||
Tasks.image_object_detection:
|
||||
(Pipelines.image_object_detection_auto,
|
||||
'damo/cv_yolox_image-object-detection-auto'),
|
||||
Tasks.ocr_recognition:
|
||||
(Pipelines.ocr_recognition,
|
||||
'damo/cv_convnextTiny_ocr-recognition-general_damo'),
|
||||
Tasks.skin_retouching: (Pipelines.skin_retouching,
|
||||
'damo/cv_unet_skin-retouching'),
|
||||
Tasks.faq_question_answering:
|
||||
(Pipelines.faq_question_answering,
|
||||
'damo/nlp_structbert_faq-question-answering_chinese-base'),
|
||||
Tasks.crowd_counting: (Pipelines.crowd_counting,
|
||||
'damo/cv_hrnet_crowd-counting_dcanet'),
|
||||
Tasks.video_single_object_tracking:
|
||||
(Pipelines.video_single_object_tracking,
|
||||
'damo/cv_vitb_video-single-object-tracking_ostrack'),
|
||||
Tasks.image_reid_person: (Pipelines.image_reid_person,
|
||||
'damo/cv_passvitb_image-reid-person_market'),
|
||||
Tasks.text_driven_segmentation:
|
||||
(Pipelines.text_driven_segmentation,
|
||||
'damo/cv_vitl16_segmentation_text-driven-seg'),
|
||||
Tasks.movie_scene_segmentation:
|
||||
(Pipelines.movie_scene_segmentation,
|
||||
'damo/cv_resnet50-bert_video-scene-segmentation_movienet'),
|
||||
Tasks.shop_segmentation: (Pipelines.shop_segmentation,
|
||||
'damo/cv_vitb16_segmentation_shop-seg'),
|
||||
Tasks.image_inpainting: (Pipelines.image_inpainting,
|
||||
'damo/cv_fft_inpainting_lama'),
|
||||
Tasks.video_inpainting: (Pipelines.video_inpainting,
|
||||
'damo/cv_video-inpainting'),
|
||||
Tasks.video_human_matting: (Pipelines.video_human_matting,
|
||||
'damo/cv_effnetv2_video-human-matting'),
|
||||
Tasks.video_frame_interpolation:
|
||||
(Pipelines.video_frame_interpolation,
|
||||
'damo/cv_raft_video-frame-interpolation'),
|
||||
Tasks.human_wholebody_keypoint:
|
||||
(Pipelines.human_wholebody_keypoint,
|
||||
'damo/cv_hrnetw48_human-wholebody-keypoint_image'),
|
||||
Tasks.hand_static: (Pipelines.hand_static,
|
||||
'damo/cv_mobileface_hand-static'),
|
||||
Tasks.face_human_hand_detection:
|
||||
(Pipelines.face_human_hand_detection,
|
||||
'damo/cv_nanodet_face-human-hand-detection'),
|
||||
Tasks.face_emotion: (Pipelines.face_emotion, 'damo/cv_face-emotion'),
|
||||
Tasks.product_segmentation: (Pipelines.product_segmentation,
|
||||
'damo/cv_F3Net_product-segmentation'),
|
||||
Tasks.referring_video_object_segmentation:
|
||||
(Pipelines.referring_video_object_segmentation,
|
||||
'damo/cv_swin-t_referring_video-object-segmentation'),
|
||||
Tasks.video_summarization: (Pipelines.video_summarization,
|
||||
'damo/cv_googlenet_pgl-video-summarization'),
|
||||
Tasks.image_skychange: (Pipelines.image_skychange,
|
||||
'damo/cv_hrnetocr_skychange'),
|
||||
Tasks.translation_evaluation: (
|
||||
Pipelines.translation_evaluation,
|
||||
'damo/nlp_unite_mup_translation_evaluation_multilingual_large'),
|
||||
Tasks.video_object_segmentation: (
|
||||
Pipelines.video_object_segmentation,
|
||||
'damo/cv_rdevos_video-object-segmentation'),
|
||||
Tasks.video_multi_object_tracking: (
|
||||
Pipelines.video_multi_object_tracking,
|
||||
'damo/cv_yolov5_video-multi-object-tracking_fairmot'),
|
||||
Tasks.image_multi_view_depth_estimation: (
|
||||
Pipelines.image_multi_view_depth_estimation,
|
||||
'damo/cv_casmvs_multi-view-depth-estimation_general'),
|
||||
Tasks.image_fewshot_detection: (
|
||||
Pipelines.image_fewshot_detection,
|
||||
'damo/cv_resnet101_detection_fewshot-defrcn'),
|
||||
Tasks.image_body_reshaping: (Pipelines.image_body_reshaping,
|
||||
'damo/cv_flow-based-body-reshaping_damo'),
|
||||
Tasks.image_face_fusion: (Pipelines.image_face_fusion,
|
||||
'damo/cv_unet-image-face-fusion_damo'),
|
||||
Tasks.image_matching: (
|
||||
Pipelines.image_matching,
|
||||
'damo/cv_quadtree_attention_image-matching_outdoor'),
|
||||
}
|
||||
|
||||
|
||||
class CVTrainers(object):
|
||||
# cv trainers
|
||||
image_instance_segmentation = 'image-instance-segmentation'
|
||||
|
||||
@@ -4,7 +4,7 @@ import os
|
||||
from typing import List, Optional, Union
|
||||
|
||||
from modelscope.hub.snapshot_download import snapshot_download
|
||||
from modelscope.metainfo import Pipelines
|
||||
from modelscope.metainfo import DEFAULT_MODEL_FOR_PIPELINE, Pipelines
|
||||
from modelscope.models.base import Model
|
||||
from modelscope.utils.config import ConfigDict, check_config
|
||||
from modelscope.utils.constant import DEFAULT_MODEL_REVISION, Invoke, Tasks
|
||||
@@ -15,271 +15,6 @@ from .util import is_official_hub_path
|
||||
|
||||
PIPELINES = Registry('pipelines')
|
||||
|
||||
DEFAULT_MODEL_FOR_PIPELINE = {
|
||||
# TaskName: (pipeline_module_name, model_repo)
|
||||
Tasks.sentence_embedding:
|
||||
(Pipelines.sentence_embedding,
|
||||
'damo/nlp_corom_sentence-embedding_english-base'),
|
||||
Tasks.text_ranking: (Pipelines.text_ranking,
|
||||
'damo/nlp_corom_passage-ranking_english-base'),
|
||||
Tasks.text_ranking: (Pipelines.mgeo_ranking,
|
||||
'damo/mgeo_address_ranking_chinese_base'),
|
||||
Tasks.word_segmentation:
|
||||
(Pipelines.word_segmentation,
|
||||
'damo/nlp_structbert_word-segmentation_chinese-base'),
|
||||
Tasks.part_of_speech: (Pipelines.part_of_speech,
|
||||
'damo/nlp_structbert_part-of-speech_chinese-base'),
|
||||
Tasks.token_classification:
|
||||
(Pipelines.part_of_speech,
|
||||
'damo/nlp_structbert_part-of-speech_chinese-base'),
|
||||
Tasks.named_entity_recognition:
|
||||
(Pipelines.named_entity_recognition,
|
||||
'damo/nlp_raner_named-entity-recognition_chinese-base-news'),
|
||||
Tasks.relation_extraction:
|
||||
(Pipelines.relation_extraction,
|
||||
'damo/nlp_bert_relation-extraction_chinese-base'),
|
||||
Tasks.information_extraction:
|
||||
(Pipelines.relation_extraction,
|
||||
'damo/nlp_bert_relation-extraction_chinese-base'),
|
||||
Tasks.sentence_similarity:
|
||||
(Pipelines.sentence_similarity,
|
||||
'damo/nlp_structbert_sentence-similarity_chinese-base'),
|
||||
Tasks.translation: (Pipelines.csanmt_translation,
|
||||
'damo/nlp_csanmt_translation_zh2en'),
|
||||
Tasks.nli: (Pipelines.nli, 'damo/nlp_structbert_nli_chinese-base'),
|
||||
Tasks.sentiment_classification:
|
||||
(Pipelines.sentiment_classification,
|
||||
'damo/nlp_structbert_sentiment-classification_chinese-base'
|
||||
), # TODO: revise back after passing the pr
|
||||
Tasks.portrait_matting: (Pipelines.portrait_matting,
|
||||
'damo/cv_unet_image-matting'),
|
||||
Tasks.human_detection: (Pipelines.human_detection,
|
||||
'damo/cv_resnet18_human-detection'),
|
||||
Tasks.image_object_detection: (Pipelines.object_detection,
|
||||
'damo/cv_vit_object-detection_coco'),
|
||||
Tasks.image_denoising: (Pipelines.image_denoise,
|
||||
'damo/cv_nafnet_image-denoise_sidd'),
|
||||
Tasks.image_deblurring: (Pipelines.image_deblur,
|
||||
'damo/cv_nafnet_image-deblur_gopro'),
|
||||
Tasks.video_stabilization: (Pipelines.video_stabilization,
|
||||
'damo/cv_dut-raft_video-stabilization_base'),
|
||||
Tasks.video_super_resolution:
|
||||
(Pipelines.video_super_resolution,
|
||||
'damo/cv_realbasicvsr_video-super-resolution_videolq'),
|
||||
Tasks.text_classification:
|
||||
(Pipelines.sentiment_classification,
|
||||
'damo/nlp_structbert_sentiment-classification_chinese-base'),
|
||||
Tasks.text_generation: (Pipelines.text_generation,
|
||||
'damo/nlp_palm2.0_text-generation_chinese-base'),
|
||||
Tasks.zero_shot_classification:
|
||||
(Pipelines.zero_shot_classification,
|
||||
'damo/nlp_structbert_zero-shot-classification_chinese-base'),
|
||||
Tasks.task_oriented_conversation: (Pipelines.dialog_modeling,
|
||||
'damo/nlp_space_dialog-modeling'),
|
||||
Tasks.dialog_state_tracking: (Pipelines.dialog_state_tracking,
|
||||
'damo/nlp_space_dialog-state-tracking'),
|
||||
Tasks.table_question_answering:
|
||||
(Pipelines.table_question_answering_pipeline,
|
||||
'damo/nlp-convai-text2sql-pretrain-cn'),
|
||||
Tasks.text_error_correction:
|
||||
(Pipelines.text_error_correction,
|
||||
'damo/nlp_bart_text-error-correction_chinese'),
|
||||
Tasks.image_captioning: (Pipelines.image_captioning,
|
||||
'damo/ofa_image-caption_coco_large_en'),
|
||||
Tasks.video_captioning:
|
||||
(Pipelines.video_captioning,
|
||||
'damo/multi-modal_hitea_video-captioning_base_en'),
|
||||
Tasks.image_portrait_stylization:
|
||||
(Pipelines.person_image_cartoon,
|
||||
'damo/cv_unet_person-image-cartoon_compound-models'),
|
||||
Tasks.ocr_detection: (Pipelines.ocr_detection,
|
||||
'damo/cv_resnet18_ocr-detection-line-level_damo'),
|
||||
Tasks.table_recognition:
|
||||
(Pipelines.table_recognition,
|
||||
'damo/cv_dla34_table-structure-recognition_cycle-centernet'),
|
||||
Tasks.document_vl_embedding:
|
||||
(Pipelines.document_vl_embedding,
|
||||
'damo/multi-modal_convnext-roberta-base_vldoc-embedding'),
|
||||
Tasks.license_plate_detection:
|
||||
(Pipelines.license_plate_detection,
|
||||
'damo/cv_resnet18_license-plate-detection_damo'),
|
||||
Tasks.fill_mask: (Pipelines.fill_mask, 'damo/nlp_veco_fill-mask-large'),
|
||||
Tasks.feature_extraction: (Pipelines.feature_extraction,
|
||||
'damo/pert_feature-extraction_base-test'),
|
||||
Tasks.action_recognition: (Pipelines.action_recognition,
|
||||
'damo/cv_TAdaConv_action-recognition'),
|
||||
Tasks.action_detection: (Pipelines.action_detection,
|
||||
'damo/cv_ResNetC3D_action-detection_detection2d'),
|
||||
Tasks.live_category: (Pipelines.live_category,
|
||||
'damo/cv_resnet50_live-category'),
|
||||
Tasks.video_category: (Pipelines.video_category,
|
||||
'damo/cv_resnet50_video-category'),
|
||||
Tasks.multi_modal_embedding: (Pipelines.multi_modal_embedding,
|
||||
'damo/multi-modal_clip-vit-base-patch16_zh'),
|
||||
Tasks.generative_multi_modal_embedding:
|
||||
(Pipelines.generative_multi_modal_embedding,
|
||||
'damo/multi-modal_gemm-vit-large-patch14_generative-multi-modal-embedding'
|
||||
),
|
||||
Tasks.multi_modal_similarity:
|
||||
(Pipelines.multi_modal_similarity,
|
||||
'damo/multi-modal_team-vit-large-patch14_multi-modal-similarity'),
|
||||
Tasks.visual_question_answering:
|
||||
(Pipelines.visual_question_answering,
|
||||
'damo/mplug_visual-question-answering_coco_large_en'),
|
||||
Tasks.video_question_answering:
|
||||
(Pipelines.video_question_answering,
|
||||
'damo/multi-modal_hitea_video-question-answering_base_en'),
|
||||
Tasks.video_embedding: (Pipelines.cmdssl_video_embedding,
|
||||
'damo/cv_r2p1d_video_embedding'),
|
||||
Tasks.text_to_image_synthesis:
|
||||
(Pipelines.text_to_image_synthesis,
|
||||
'damo/cv_diffusion_text-to-image-synthesis_tiny'),
|
||||
Tasks.body_2d_keypoints: (Pipelines.body_2d_keypoints,
|
||||
'damo/cv_hrnetv2w32_body-2d-keypoints_image'),
|
||||
Tasks.body_3d_keypoints: (Pipelines.body_3d_keypoints,
|
||||
'damo/cv_canonical_body-3d-keypoints_video'),
|
||||
Tasks.hand_2d_keypoints:
|
||||
(Pipelines.hand_2d_keypoints,
|
||||
'damo/cv_hrnetw18_hand-pose-keypoints_coco-wholebody'),
|
||||
Tasks.card_detection: (Pipelines.card_detection,
|
||||
'damo/cv_resnet_carddetection_scrfd34gkps'),
|
||||
Tasks.face_detection:
|
||||
(Pipelines.mog_face_detection,
|
||||
'damo/cv_resnet101_face-detection_cvpr22papermogface'),
|
||||
Tasks.face_liveness: (Pipelines.face_liveness_ir,
|
||||
'damo/cv_manual_face-liveness_flir'),
|
||||
Tasks.face_recognition: (Pipelines.face_recognition,
|
||||
'damo/cv_ir101_facerecognition_cfglint'),
|
||||
Tasks.facial_expression_recognition:
|
||||
(Pipelines.facial_expression_recognition,
|
||||
'damo/cv_vgg19_facial-expression-recognition_fer'),
|
||||
Tasks.face_attribute_recognition:
|
||||
(Pipelines.face_attribute_recognition,
|
||||
'damo/cv_resnet34_face-attribute-recognition_fairface'),
|
||||
Tasks.face_2d_keypoints: (Pipelines.face_2d_keypoints,
|
||||
'damo/cv_mobilenet_face-2d-keypoints_alignment'),
|
||||
Tasks.video_multi_modal_embedding:
|
||||
(Pipelines.video_multi_modal_embedding,
|
||||
'damo/multi_modal_clip_vtretrival_msrvtt_53'),
|
||||
Tasks.image_color_enhancement:
|
||||
(Pipelines.image_color_enhance,
|
||||
'damo/cv_csrnet_image-color-enhance-models'),
|
||||
Tasks.virtual_try_on: (Pipelines.virtual_try_on,
|
||||
'damo/cv_daflow_virtual-try-on_base'),
|
||||
Tasks.image_colorization: (Pipelines.ddcolor_image_colorization,
|
||||
'damo/cv_ddcolor_image-colorization'),
|
||||
Tasks.video_colorization: (Pipelines.video_colorization,
|
||||
'damo/cv_unet_video-colorization'),
|
||||
Tasks.image_segmentation:
|
||||
(Pipelines.image_instance_segmentation,
|
||||
'damo/cv_swin-b_image-instance-segmentation_coco'),
|
||||
Tasks.image_depth_estimation:
|
||||
(Pipelines.image_depth_estimation,
|
||||
'damo/cv_newcrfs_image-depth-estimation_indoor'),
|
||||
Tasks.indoor_layout_estimation:
|
||||
(Pipelines.indoor_layout_estimation,
|
||||
'damo/cv_panovit_indoor-layout-estimation'),
|
||||
Tasks.video_depth_estimation:
|
||||
(Pipelines.video_depth_estimation,
|
||||
'damo/cv_dro-resnet18_video-depth-estimation_indoor'),
|
||||
Tasks.panorama_depth_estimation:
|
||||
(Pipelines.panorama_depth_estimation,
|
||||
'damo/cv_unifuse_panorama-depth-estimation'),
|
||||
Tasks.image_style_transfer: (Pipelines.image_style_transfer,
|
||||
'damo/cv_aams_style-transfer_damo'),
|
||||
Tasks.face_image_generation: (Pipelines.face_image_generation,
|
||||
'damo/cv_gan_face-image-generation'),
|
||||
Tasks.image_super_resolution: (Pipelines.image_super_resolution,
|
||||
'damo/cv_rrdb_image-super-resolution'),
|
||||
Tasks.image_portrait_enhancement:
|
||||
(Pipelines.image_portrait_enhancement,
|
||||
'damo/cv_gpen_image-portrait-enhancement'),
|
||||
Tasks.product_retrieval_embedding:
|
||||
(Pipelines.product_retrieval_embedding,
|
||||
'damo/cv_resnet50_product-bag-embedding-models'),
|
||||
Tasks.image_to_image_generation:
|
||||
(Pipelines.image_to_image_generation,
|
||||
'damo/cv_latent_diffusion_image2image_generate'),
|
||||
Tasks.image_classification:
|
||||
(Pipelines.daily_image_classification,
|
||||
'damo/cv_vit-base_image-classification_Dailylife-labels'),
|
||||
Tasks.image_object_detection:
|
||||
(Pipelines.image_object_detection_auto,
|
||||
'damo/cv_yolox_image-object-detection-auto'),
|
||||
Tasks.ocr_recognition:
|
||||
(Pipelines.ocr_recognition,
|
||||
'damo/cv_convnextTiny_ocr-recognition-general_damo'),
|
||||
Tasks.skin_retouching: (Pipelines.skin_retouching,
|
||||
'damo/cv_unet_skin-retouching'),
|
||||
Tasks.faq_question_answering:
|
||||
(Pipelines.faq_question_answering,
|
||||
'damo/nlp_structbert_faq-question-answering_chinese-base'),
|
||||
Tasks.crowd_counting: (Pipelines.crowd_counting,
|
||||
'damo/cv_hrnet_crowd-counting_dcanet'),
|
||||
Tasks.video_single_object_tracking:
|
||||
(Pipelines.video_single_object_tracking,
|
||||
'damo/cv_vitb_video-single-object-tracking_ostrack'),
|
||||
Tasks.image_reid_person: (Pipelines.image_reid_person,
|
||||
'damo/cv_passvitb_image-reid-person_market'),
|
||||
Tasks.text_driven_segmentation:
|
||||
(Pipelines.text_driven_segmentation,
|
||||
'damo/cv_vitl16_segmentation_text-driven-seg'),
|
||||
Tasks.movie_scene_segmentation:
|
||||
(Pipelines.movie_scene_segmentation,
|
||||
'damo/cv_resnet50-bert_video-scene-segmentation_movienet'),
|
||||
Tasks.shop_segmentation: (Pipelines.shop_segmentation,
|
||||
'damo/cv_vitb16_segmentation_shop-seg'),
|
||||
Tasks.image_inpainting: (Pipelines.image_inpainting,
|
||||
'damo/cv_fft_inpainting_lama'),
|
||||
Tasks.video_inpainting: (Pipelines.video_inpainting,
|
||||
'damo/cv_video-inpainting'),
|
||||
Tasks.video_human_matting: (Pipelines.video_human_matting,
|
||||
'damo/cv_effnetv2_video-human-matting'),
|
||||
Tasks.video_frame_interpolation:
|
||||
(Pipelines.video_frame_interpolation,
|
||||
'damo/cv_raft_video-frame-interpolation'),
|
||||
Tasks.human_wholebody_keypoint:
|
||||
(Pipelines.human_wholebody_keypoint,
|
||||
'damo/cv_hrnetw48_human-wholebody-keypoint_image'),
|
||||
Tasks.hand_static: (Pipelines.hand_static,
|
||||
'damo/cv_mobileface_hand-static'),
|
||||
Tasks.face_human_hand_detection:
|
||||
(Pipelines.face_human_hand_detection,
|
||||
'damo/cv_nanodet_face-human-hand-detection'),
|
||||
Tasks.face_emotion: (Pipelines.face_emotion, 'damo/cv_face-emotion'),
|
||||
Tasks.product_segmentation: (Pipelines.product_segmentation,
|
||||
'damo/cv_F3Net_product-segmentation'),
|
||||
Tasks.referring_video_object_segmentation:
|
||||
(Pipelines.referring_video_object_segmentation,
|
||||
'damo/cv_swin-t_referring_video-object-segmentation'),
|
||||
Tasks.video_summarization: (Pipelines.video_summarization,
|
||||
'damo/cv_googlenet_pgl-video-summarization'),
|
||||
Tasks.image_skychange: (Pipelines.image_skychange,
|
||||
'damo/cv_hrnetocr_skychange'),
|
||||
Tasks.translation_evaluation: (
|
||||
Pipelines.translation_evaluation,
|
||||
'damo/nlp_unite_mup_translation_evaluation_multilingual_large'),
|
||||
Tasks.video_object_segmentation: (
|
||||
Pipelines.video_object_segmentation,
|
||||
'damo/cv_rdevos_video-object-segmentation'),
|
||||
Tasks.video_multi_object_tracking: (
|
||||
Pipelines.video_multi_object_tracking,
|
||||
'damo/cv_yolov5_video-multi-object-tracking_fairmot'),
|
||||
Tasks.image_multi_view_depth_estimation: (
|
||||
Pipelines.image_multi_view_depth_estimation,
|
||||
'damo/cv_casmvs_multi-view-depth-estimation_general'),
|
||||
Tasks.image_fewshot_detection: (
|
||||
Pipelines.image_fewshot_detection,
|
||||
'damo/cv_resnet101_detection_fewshot-defrcn'),
|
||||
Tasks.image_body_reshaping: (Pipelines.image_body_reshaping,
|
||||
'damo/cv_flow-based-body-reshaping_damo'),
|
||||
Tasks.image_face_fusion: (Pipelines.image_face_fusion,
|
||||
'damo/cv_unet-image-face-fusion_damo'),
|
||||
Tasks.image_matching: (
|
||||
Pipelines.image_matching,
|
||||
'damo/cv_quadtree_attention_image-matching_outdoor'),
|
||||
}
|
||||
|
||||
|
||||
def normalize_model_input(model, model_revision):
|
||||
""" normalize the input model, to ensure that a model str is a valid local path: in other words,
|
||||
|
||||
@@ -378,6 +378,11 @@ def get_selected_cases():
|
||||
if line.startswith('Selected cases:'):
|
||||
line = line.replace('Selected cases:', '').strip()
|
||||
selected_cases = line.split(',')
|
||||
sub_process.wait()
|
||||
if sub_process.returncode != 0:
|
||||
msg = 'Run analysis exception, returncode: %s!' % sub_process.returncode
|
||||
logger.error(msg)
|
||||
raise Exception(msg)
|
||||
return selected_cases
|
||||
|
||||
|
||||
@@ -390,7 +395,8 @@ def run_in_subprocess(args):
|
||||
for f in test_suite_files:
|
||||
logger.info(f)
|
||||
except Exception:
|
||||
logger.error('Get test suite based diff exception!')
|
||||
logger.error(
|
||||
'Get test suite based diff exception!, will run all cases.')
|
||||
test_suite_files = gather_test_suites_files(
|
||||
os.path.abspath(args.test_dir), args.pattern)
|
||||
if len(test_suite_files) == 0:
|
||||
|
||||
@@ -40,21 +40,27 @@ def get_models_info(groups: list) -> dict:
|
||||
configuration_file = os.path.join(cache_root, model_id,
|
||||
ModelFile.CONFIGURATION)
|
||||
if not os.path.exists(configuration_file):
|
||||
model_revisions = api.list_model_revisions(model_id=model_id)
|
||||
if len(model_revisions) == 0:
|
||||
logger.warn('Model: %s has no revision' % model_id)
|
||||
continue
|
||||
# get latest revision
|
||||
try:
|
||||
model_revisions = api.list_model_revisions(model_id=model_id)
|
||||
if len(model_revisions) == 0:
|
||||
print('Model: %s has no revision' % model_id)
|
||||
continue
|
||||
# get latest revision
|
||||
configuration_file = model_file_download(
|
||||
model_id=model_id,
|
||||
file_path=ModelFile.CONFIGURATION,
|
||||
revision=model_revisions[0])
|
||||
except NotExistError:
|
||||
logger.warn('Model: %s has no configuration file %s' %
|
||||
(model_id, ModelFile.CONFIGURATION))
|
||||
except Exception as e:
|
||||
print('Download model: %s configuration file exception'
|
||||
% model_id)
|
||||
print('Exception: %s' % e)
|
||||
continue
|
||||
cfg = Config.from_file(configuration_file)
|
||||
try:
|
||||
cfg = Config.from_file(configuration_file)
|
||||
except Exception as e:
|
||||
print('Resolve model: %s configuration file failed!' % model_id)
|
||||
print(('Exception: %s' % e))
|
||||
|
||||
model_info = {}
|
||||
model_info['framework'] = cfg.safe_get('framework')
|
||||
model_info['task'] = cfg.safe_get('task')
|
||||
@@ -83,13 +89,17 @@ def get_models_info(groups: list) -> dict:
|
||||
return models_info
|
||||
|
||||
|
||||
def gather_test_suites_files_full_path(test_dir='./tests',
|
||||
pattern='test_*.py'):
|
||||
def gather_test_suites_files(test_dir='./tests',
|
||||
pattern='test_*.py',
|
||||
is_full_path=True):
|
||||
case_file_list = []
|
||||
for dirpath, dirnames, filenames in os.walk(test_dir):
|
||||
for file in filenames:
|
||||
if fnmatch(file, pattern):
|
||||
case_file_list.append(os.path.join(dirpath, file))
|
||||
if is_full_path:
|
||||
case_file_list.append(os.path.join(dirpath, file))
|
||||
else:
|
||||
case_file_list.append(file)
|
||||
|
||||
return case_file_list
|
||||
|
||||
@@ -102,12 +112,18 @@ def run_command_get_output(cmd):
|
||||
output = response.stdout.decode('utf8')
|
||||
return output
|
||||
except subprocess.CalledProcessError as error:
|
||||
logger.error(
|
||||
'stdout: %s, stderr: %s' %
|
||||
(response.stdout.decode('utf8'), error.stderr.decode('utf8')))
|
||||
print('stdout: %s, stderr: %s' %
|
||||
(response.stdout.decode('utf8'), error.stderr.decode('utf8')))
|
||||
return None
|
||||
|
||||
|
||||
def get_current_branch():
|
||||
cmd = ['git', 'rev-parse', '--abbrev-ref', 'HEAD']
|
||||
branch = run_command_get_output(cmd).strip()
|
||||
logger.info('Testing branch: %s' % branch)
|
||||
return branch
|
||||
|
||||
|
||||
def get_modified_files():
|
||||
cmd = ['git', 'diff', '--name-only', 'origin/master...']
|
||||
cmd_output = run_command_get_output(cmd)
|
||||
@@ -120,7 +136,10 @@ def analysis_diff():
|
||||
"""Get modified files and their imported files modified modules
|
||||
"""
|
||||
# ignore diff for constant define files, these files import by all pipeline, trainer
|
||||
ignore_files = ['modelscope/utils/constant.py', 'modelscope/metainfo.py']
|
||||
ignore_files = [
|
||||
'modelscope/utils/constant.py', 'modelscope/metainfo.py',
|
||||
'modelscope/pipeline_inputs.py', 'modelscope/outputs/outputs.py'
|
||||
]
|
||||
|
||||
modified_register_modules = []
|
||||
modified_cases = []
|
||||
@@ -157,7 +176,7 @@ def analysis_diff():
|
||||
|
||||
|
||||
def split_test_suites():
|
||||
test_suite_full_paths = gather_test_suites_files_full_path()
|
||||
test_suite_full_paths = gather_test_suites_files()
|
||||
pipeline_test_suites = []
|
||||
trainer_test_suites = []
|
||||
other_test_suites = []
|
||||
@@ -173,6 +192,10 @@ def split_test_suites():
|
||||
|
||||
|
||||
def get_test_suites_to_run():
|
||||
branch = get_current_branch()
|
||||
if branch == 'master':
|
||||
# when run with master, run all the cases
|
||||
return gather_test_suites_files(is_full_path=False)
|
||||
affected_register_modules, modified_cases = analysis_diff()
|
||||
# affected_register_modules list of modified file and dependent file's register_module.
|
||||
# ("MODULES|PIPELINES|TRAINERS|...", '', '', model_class_name)
|
||||
@@ -219,10 +242,10 @@ def get_test_suites_to_run():
|
||||
elif affected_register_module[0] == 'PREPROCESSORS':
|
||||
# ["PREPROCESSORS", "cv", "object_detection_scrfd", "SCRFDPreprocessor"]
|
||||
# ["PREPROCESSORS", domain, preprocessor_name, class_name]
|
||||
task = model_info['task']
|
||||
for model_id, model_info in models_info.items():
|
||||
if model_info['preprocessor_type'] is not None and model_info[
|
||||
'preprocessor_type'] == affected_register_module[2]:
|
||||
task = model_info['task']
|
||||
if task in task_pipeline_test_suite_map:
|
||||
affected_pipeline_cases.extend(
|
||||
task_pipeline_test_suite_map[task])
|
||||
|
||||
@@ -219,7 +219,6 @@ class AnalysisSourceFileRegisterModules(ast.NodeVisitor):
|
||||
def get_imported_files(file_path):
|
||||
"""Get file dependencies.
|
||||
"""
|
||||
print('Getting %s imports' % file_path)
|
||||
if os.path.isabs(file_path):
|
||||
file_path = os.path.relpath(file_path, os.getcwd())
|
||||
with open(file_path, 'rb') as f:
|
||||
|
||||
Reference in New Issue
Block a user