From 4225048dbf4c6bcfa33e52d9895340fe9385dda6 Mon Sep 17 00:00:00 2001 From: "liugao.lg" Date: Tue, 18 Apr 2023 14:41:51 +0800 Subject: [PATCH 01/76] provide allow_remote MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 优化模版文件，提升独立repo接入成功率，降低用户阅读文档带来的歧义 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12354329 * provide allow_remote --- modelscope/cli/template/template.tpl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modelscope/cli/template/template.tpl b/modelscope/cli/template/template.tpl index 0c09a925..78fe339c 100644 --- a/modelscope/cli/template/template.tpl +++ b/modelscope/cli/template/template.tpl @@ -122,10 +122,11 @@ class ${pipeline_name}(Pipeline): # Tips: usr_config_path is the temporary save configuration location， after upload modelscope hub, it is the model_id usr_config_path = '${configuration_path}' config = Config({ - 'framework': 'pytorch', - 'task': '${task_name}', - 'model': {'type': 'my-custom-model'}, - "pipeline": {"type": "my-custom-pipeline"} + "framework": 'pytorch', + "task": '${task_name}', + "model": {'type': 'my-custom-model'}, + "pipeline": {"type": "my-custom-pipeline"}, + "allow_remote": True }) config.dump('${configuration_path}' + 'configuration.json') From 3033e54d5e462fdf628a0a628d0fb09346a452b1 Mon Sep 17 00:00:00 2001 From: "biwen.lbw" Date: Tue, 18 Apr 2023 14:42:09 +0800 Subject: [PATCH 02/76] update test data Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12355868 * update test data --- data/test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/test b/data/test index d0cd4f21..c117008c 160000 --- a/data/test +++ b/data/test @@ -1 +1 @@ -Subproject commit d0cd4f218f200267a7b24ae38df53e47cb15bf2d +Subproject commit c117008caa9dc447c208e9ed6bc11310512d4a3a From a355eefbd4429478c88d3eb148019082381ddde8 Mon Sep 17 00:00:00 2001 From: "shuying.shu" Date: Tue, 18 Apr 2023 14:43:09 +0800 Subject: [PATCH 03/76] fix OOM for bassl Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12354474 --- .../cv/movie_scene_segmentation/model.py | 86 +++++++++++++------ .../movie_scene_segmentation/utils/save_op.py | 5 +- .../cv/movie_scene_segmentation_pipeline.py | 23 +++-- requirements/cv.txt | 2 +- 4 files changed, 78 insertions(+), 38 deletions(-) diff --git a/modelscope/models/cv/movie_scene_segmentation/model.py b/modelscope/models/cv/movie_scene_segmentation/model.py index 336af3b3..818a3876 100644 --- a/modelscope/models/cv/movie_scene_segmentation/model.py +++ b/modelscope/models/cv/movie_scene_segmentation/model.py @@ -13,7 +13,8 @@ import torch.nn as nn import torch.nn.functional as F import torchvision.transforms as TF from PIL import Image -from shotdetect_scenedetect_lgss import shot_detect +from shotdetect_scenedetect_lgss import shot_detector +from tqdm import tqdm from modelscope.metainfo import Models from modelscope.models.base.base_torch_model import TorchModel @@ -60,6 +61,9 @@ class MovieSceneSegmentationModel(TorchModel): self.head_sbd = nn.Linear(hdim, 2) load_param_with_prefix('head_sbd', self.head_sbd, params) + self.shot_detector = shot_detector() + self.shot_detector.init(**self.cfg.preprocessor.shot_detect) + self.test_transform = TF.Compose([ TF.Resize(size=256, interpolation=Image.BICUBIC), TF.CenterCrop(224), @@ -98,29 +102,45 @@ class MovieSceneSegmentationModel(TorchModel): def inference(self, batch): logger.info('Begin scene detect ......') bs = self.cfg.pipeline.batch_size_per_gpu - sids = batch['sid'] - inputs = batch['shot_feat'] + device = self.crn.attention_mask.device - shot_num = len(sids) + shot_timecode_lst = batch['shot_timecode_lst'] + shot_idx_lst = batch['shot_idx_lst'] + + shot_num = len(shot_timecode_lst) cnt = math.ceil(shot_num / bs) - infer_sid, infer_pred = [], [] + infer_pred = [] infer_result = {} - for i in range(cnt): + self.shot_detector.start() + + for i in tqdm(range(cnt)): start = i * bs end = (i + 1) * bs if (i + 1) * bs < shot_num else shot_num - input_ = inputs[start:end] - sid_ = sids[start:end] - input_ = torch.stack(input_) + + batch_shot_idx_lst = shot_idx_lst[start:end] + + shot_start_idx = batch_shot_idx_lst[0][0] + shot_end_idx = batch_shot_idx_lst[-1][-1] + batch_timecode_lst = { + i: shot_timecode_lst[i] + for i in range(shot_start_idx, shot_end_idx + 1) + } + batch_shot_keyf_lst = self.shot_detector.get_frame_img( + batch_timecode_lst, shot_start_idx, shot_num) + inputs = self.get_batch_input(batch_shot_keyf_lst, shot_start_idx, + batch_shot_idx_lst) + + input_ = torch.stack(inputs).to(device) outputs = self.shared_step(input_) # shape [b,2] prob = F.softmax(outputs, dim=1) - infer_sid.extend(sid_.cpu().detach().numpy()) infer_pred.extend(prob[:, 1].cpu().detach().numpy()) - infer_result.update({'pred': np.stack(infer_pred)}) - infer_result.update({'sid': infer_sid}) - assert len(infer_result['sid']) == len(sids) - assert len(infer_result['pred']) == len(inputs) + infer_result.update({'pred': np.stack(infer_pred)}) + infer_result.update({'sid': np.arange(shot_num)}) + + assert len(infer_result['pred']) == shot_num + self.shot_detector.release() return infer_result def shared_step(self, inputs): @@ -162,38 +182,48 @@ class MovieSceneSegmentationModel(TorchModel): logger.info('Generate scene .......') pred_dict = inputs['feat'] + shot2keyf = inputs['shot2keyf'] thres = self.cfg.pipeline.save_threshold anno_dict = get_pred_boundary(pred_dict, thres) scene_dict_lst, scene_list, shot_num, shot_dict_lst = pred2scene( - self.shot2keyf, anno_dict) + shot2keyf, anno_dict) if self.cfg.pipeline.save_split_scene: re_dir = scene2video(inputs['input_video_pth'], scene_list, thres) print(f'Split scene video saved to {re_dir}') return len(scene_list), scene_dict_lst, shot_num, shot_dict_lst - def preprocess(self, inputs): - logger.info('Begin shot detect......') - shot_keyf_lst, anno, shot2keyf = shot_detect( - inputs, **self.cfg.preprocessor.shot_detect) - logger.info('Shot detect done!') + def get_batch_input(self, shot_keyf_lst, shot_start_idx, shot_idx_lst): - single_shot_feat, sid = [], [] + single_shot_feat = [] for idx, one_shot in enumerate(shot_keyf_lst): one_shot = [ self.test_transform(one_frame) for one_frame in one_shot ] one_shot = torch.stack(one_shot, dim=0) single_shot_feat.append(one_shot) - sid.append(idx) + single_shot_feat = torch.stack(single_shot_feat, dim=0) + shot_feat = [] + for idx, shot_idx in enumerate(shot_idx_lst): + shot_idx_ = shot_idx - shot_start_idx + _one_shot = single_shot_feat[shot_idx_] + shot_feat.append(_one_shot) + + return shot_feat + + def preprocess(self, inputs): + logger.info('Begin shot detect......') + shot_timecode_lst, anno, shot2keyf = self.shot_detector.shot_detect( + inputs, **self.cfg.preprocessor.shot_detect) + logger.info('Shot detect done!') + + shot_idx_lst = [] for idx, one_shot in enumerate(anno): shot_idx = int(one_shot['shot_id']) + np.arange( -self.neighbor_size, self.neighbor_size + 1) - shot_idx = np.clip(shot_idx, 0, one_shot['num_shot']) - _one_shot = single_shot_feat[shot_idx] - shot_feat.append(_one_shot) - self.shot2keyf = shot2keyf - self.anno = anno - return shot_feat, sid + shot_idx = np.clip(shot_idx, 0, one_shot['num_shot'] - 1) + shot_idx_lst.append(shot_idx) + + return shot2keyf, anno, shot_timecode_lst, shot_idx_lst diff --git a/modelscope/models/cv/movie_scene_segmentation/utils/save_op.py b/modelscope/models/cv/movie_scene_segmentation/utils/save_op.py index 49155716..34bebce0 100644 --- a/modelscope/models/cv/movie_scene_segmentation/utils/save_op.py +++ b/modelscope/models/cv/movie_scene_segmentation/utils/save_op.py @@ -10,11 +10,12 @@ from tqdm import tqdm def get_pred_boundary(pred_dict, threshold=0.5): - pred = pred_dict['pred'] + pred = pred_dict['pred'].cpu().numpy() + sid = pred_dict['sid'].cpu().numpy().astype(np.int32) tmp = (pred > threshold).astype(np.int32) anno_dict = {} for idx in range(len(tmp)): - anno_dict.update({str(pred_dict['sid'][idx]).zfill(4): int(tmp[idx])}) + anno_dict.update({str(sid[idx]).zfill(4): int(tmp[idx])}) return anno_dict diff --git a/modelscope/pipelines/cv/movie_scene_segmentation_pipeline.py b/modelscope/pipelines/cv/movie_scene_segmentation_pipeline.py index 3fffc546..3cef5c28 100644 --- a/modelscope/pipelines/cv/movie_scene_segmentation_pipeline.py +++ b/modelscope/pipelines/cv/movie_scene_segmentation_pipeline.py @@ -43,23 +43,32 @@ class MovieSceneSegmentationPipeline(Pipeline): """ self.input_video_pth = input if isinstance(input, str): - shot_feat, sid = self.model.preprocess(input) + self.shot2keyf, self.anno, self.shot_timecode_lst, self.shot_idx_lst = self.model.preprocess( + input) else: raise TypeError(f'input should be a str,' f' but got {type(input)}') - result = {'sid': sid, 'shot_feat': shot_feat} + result = { + 'shot_timecode_lst': self.shot_timecode_lst, + 'shot_idx_lst': self.shot_idx_lst + } - return result + with torch.no_grad(): + output = self.model.inference(result) + + return output def forward(self, input: Dict[str, Any], **forward_params) -> Dict[str, Any]: - with torch.no_grad(): - output = self.model.inference(input) - return output + return input def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: - data = {'input_video_pth': self.input_video_pth, 'feat': inputs} + data = { + 'input_video_pth': self.input_video_pth, + 'feat': inputs, + 'shot2keyf': self.shot2keyf + } scene_num, scene_meta_lst, shot_num, shot_meta_lst = self.model.postprocess( data) result = { diff --git a/requirements/cv.txt b/requirements/cv.txt index 7d09a60b..1c9ed435 100644 --- a/requirements/cv.txt +++ b/requirements/cv.txt @@ -51,7 +51,7 @@ regex scikit-image>=0.19.3 scikit-learn>=0.20.1 shapely -shotdetect_scenedetect_lgss +shotdetect_scenedetect_lgss>=0.0.4 smplx tensorflow-estimator>=1.15.1 tf_slim From 976bdf4ebc33e5439c7d6618751badfa866ec4ed Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Tue, 18 Apr 2023 17:01:43 +0800 Subject: [PATCH 04/76] Add virgo hub 1. Add Hubs.virgo in MsDataset.load() 2. Add VirgoDownloader pipeline 3. Add auth for virgo hub. 4. Add other utils. 5. cherry-pick from https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12352198?file=8c317278a40b19fe76825fb749e768fdbfd6a711 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12351633 --- modelscope/hub/api.py | 45 +++++- modelscope/msdatasets/auth/auth_config.py | 9 ++ .../context/dataset_context_config.py | 10 ++ .../msdatasets/data_loader/data_loader.py | 150 ++++++++++++++++-- .../data_loader/data_loader_manager.py | 4 +- modelscope/msdatasets/dataset_cls/__init__.py | 2 +- modelscope/msdatasets/dataset_cls/dataset.py | 81 +++++++++- .../msdatasets/meta/data_meta_manager.py | 8 + modelscope/msdatasets/ms_dataset.py | 30 +++- modelscope/utils/constant.py | 21 +++ modelscope/utils/service_utils.py | 5 + modelscope/utils/url_utils.py | 36 +++++ tests/msdatasets/test_virgo_dataset.py | 76 +++++++++ 13 files changed, 450 insertions(+), 27 deletions(-) create mode 100644 modelscope/utils/url_utils.py create mode 100644 tests/msdatasets/test_virgo_dataset.py diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index ad8d0c5d..0991dbb9 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -15,6 +15,7 @@ from http.cookiejar import CookieJar from os.path import expanduser from typing import Dict, List, Optional, Tuple, Union +import requests from requests import Session from requests.adapters import HTTPAdapter, Retry @@ -45,7 +46,7 @@ from modelscope.utils.constant import (DEFAULT_DATASET_REVISION, MASTER_MODEL_BRANCH, DatasetFormations, DatasetMetaFormats, DatasetVisibilityMap, DownloadChannel, - ModelFile) + ModelFile, VirgoDatasetConfig) from modelscope.utils.logger import get_logger from .utils.utils import (get_endpoint, get_release_datetime, model_id_to_group_owner_name) @@ -581,6 +582,17 @@ class HubApi: file_list = file_list['Files'] return file_list + @staticmethod + def dump_datatype_file(dataset_type: int, meta_cache_dir: str): + """ + Dump the data_type as a local file, in order to get the dataset formation without calling the datahub. + More details, please refer to the class `modelscope.utils.constant.DatasetFormations`. + """ + dataset_type_file_path = os.path.join(meta_cache_dir, + f'{str(dataset_type)}{DatasetFormations.formation_mark_ext.value}') + with open(dataset_type_file_path, 'w') as fp: + fp.write('*** Automatically-generated file, do not modify ***') + def get_dataset_meta_files_local_paths(self, dataset_name: str, namespace: str, revision: str, @@ -591,10 +603,7 @@ class HubApi: cookies = ModelScopeConfig.get_cookies() # Dump the data_type as a local file - dataset_type_file_path = os.path.join(meta_cache_dir, - f'{str(dataset_type)}{DatasetFormations.formation_mark_ext.value}') - with open(dataset_type_file_path, 'w') as fp: - fp.write('*** Automatically-generated file, do not modify ***') + HubApi.dump_datatype_file(dataset_type=dataset_type, meta_cache_dir=meta_cache_dir) for file_info in file_list: file_path = file_info['Path'] @@ -661,7 +670,6 @@ class HubApi: cookies = self._check_cookie(use_cookies=True) else: cookies = ModelScopeConfig.get_cookies() - r = self.session.get(url=datahub_url, cookies=cookies, headers=self.headers) r = self.session.get( url=datahub_url, cookies=cookies, headers=self.headers) @@ -669,6 +677,31 @@ class HubApi: raise_on_error(resp) return resp['Data'] + def get_virgo_meta(self, dataset_id: str, version: int = 1) -> dict: + """ + Get virgo dataset meta info. + """ + virgo_endpoint = os.environ.get(VirgoDatasetConfig.env_virgo_endpoint, '') + if not virgo_endpoint: + raise RuntimeError(f'Virgo endpoint is not set in env: {VirgoDatasetConfig.env_virgo_endpoint}') + + virgo_dataset_url = f'{virgo_endpoint}/data/set/download' + cookies = requests.utils.dict_from_cookiejar(ModelScopeConfig.get_cookies()) + + dataset_info = dict( + dataSetId=dataset_id, + dataSetVersion=version + ) + data = dict( + data=dataset_info, + ) + r = self.session.post(url=virgo_dataset_url, json=data, cookies=cookies, headers=self.headers, timeout=900) + resp = r.json() + if resp['code'] != 0: + raise RuntimeError(f'Failed to get virgo dataset: {resp}') + + return resp['data'] + def get_dataset_access_config_for_unzipped(self, dataset_name: str, namespace: str, diff --git a/modelscope/msdatasets/auth/auth_config.py b/modelscope/msdatasets/auth/auth_config.py index 576a6efd..e09db93c 100644 --- a/modelscope/msdatasets/auth/auth_config.py +++ b/modelscope/msdatasets/auth/auth_config.py @@ -23,6 +23,15 @@ class OssAuthConfig(BaseAuthConfig): cookies=cookies, git_token=git_token, user_info=user_info) +class VirgoAuthConfig(BaseAuthConfig): + """The authorization config for virgo dataset.""" + + def __init__(self, cookies: CookieJar, git_token: str, + user_info: Tuple[str, str]): + super().__init__( + cookies=cookies, git_token=git_token, user_info=user_info) + + class MaxComputeAuthConfig(BaseAuthConfig): # TODO: MaxCompute dataset to be supported. def __init__(self, cookies: CookieJar, git_token: str, diff --git a/modelscope/msdatasets/context/dataset_context_config.py b/modelscope/msdatasets/context/dataset_context_config.py index 26b05f7d..929e68d0 100644 --- a/modelscope/msdatasets/context/dataset_context_config.py +++ b/modelscope/msdatasets/context/dataset_context_config.py @@ -24,6 +24,7 @@ class DatasetContextConfig: self._config_kwargs = kwargs self._dataset_version_cache_root_dir = None self._auth_config = None + self._ext_config: dict = {} # The lock file path for meta-files and data-files self._global_meta_lock_file_path = None @@ -42,6 +43,7 @@ class DatasetContextConfig: self.data_files = data_files self.cache_root_dir = cache_root_dir self.use_streaming = use_streaming + self.download_virgo_files: bool = False @property def config_kwargs(self) -> dict: @@ -98,3 +100,11 @@ class DatasetContextConfig: @auth_config.setter def auth_config(self, val: BaseAuthConfig): self._auth_config = val + + @property + def ext_config(self) -> dict: + return self._ext_config + + @ext_config.setter + def ext_config(self, val: dict): + self._ext_config = val diff --git a/modelscope/msdatasets/data_loader/data_loader.py b/modelscope/msdatasets/data_loader/data_loader.py index 1ef92372..38b83c46 100644 --- a/modelscope/msdatasets/data_loader/data_loader.py +++ b/modelscope/msdatasets/data_loader/data_loader.py @@ -1,11 +1,12 @@ # Copyright (c) Alibaba, Inc. and its affiliates. +import os from abc import ABC, abstractmethod from typing import Optional, Union from datasets import (Dataset, DatasetBuilder, DatasetDict, IterableDataset, IterableDatasetDict) -from datasets import load_dataset as hf_data_loader +from datasets import load_dataset as hf_load_dataset from modelscope.hub.api import ModelScopeConfig from modelscope.msdatasets.auth.auth_config import OssAuthConfig @@ -15,11 +16,16 @@ from modelscope.msdatasets.data_files.data_files_manager import \ DataFilesManager from modelscope.msdatasets.dataset_cls.dataset import ExternalDataset from modelscope.msdatasets.meta.data_meta_manager import DataMetaManager -from modelscope.utils.constant import DatasetFormations +from modelscope.utils.constant import (DatasetFormations, DatasetPathName, + DownloadMode, VirgoDatasetConfig) +from modelscope.utils.logger import get_logger +from modelscope.utils.url_utils import valid_url + +logger = get_logger() -class BaseDataLoader(ABC): - """Base dataset loader to load data.""" +class BaseDownloader(ABC): + """Base dataset downloader to load data.""" def __init__(self, dataset_context_config: DatasetContextConfig): self.dataset_context_config = dataset_context_config @@ -28,35 +34,35 @@ class BaseDataLoader(ABC): def process(self): """The entity processing pipeline for fetching the data. """ raise NotImplementedError( - f'No default implementation provided for {BaseDataLoader.__name__}.process.' + f'No default implementation provided for {BaseDownloader.__name__}.process.' ) @abstractmethod def _authorize(self): raise NotImplementedError( - f'No default implementation provided for {BaseDataLoader.__name__}._authorize.' + f'No default implementation provided for {BaseDownloader.__name__}._authorize.' ) @abstractmethod def _build(self): raise NotImplementedError( - f'No default implementation provided for {BaseDataLoader.__name__}._build.' + f'No default implementation provided for {BaseDownloader.__name__}._build.' ) @abstractmethod def _prepare_and_download(self): raise NotImplementedError( - f'No default implementation provided for {BaseDataLoader.__name__}._prepare_and_download.' + f'No default implementation provided for {BaseDownloader.__name__}._prepare_and_download.' ) @abstractmethod def _post_process(self): raise NotImplementedError( - f'No default implementation provided for {BaseDataLoader.__name__}._post_process.' + f'No default implementation provided for {BaseDownloader.__name__}._post_process.' ) -class OssDataLoader(BaseDataLoader): +class OssDownloader(BaseDownloader): def __init__(self, dataset_context_config: DatasetContextConfig): super().__init__(dataset_context_config) @@ -127,7 +133,7 @@ class OssDataLoader(BaseDataLoader): raise f'meta-file: {dataset_name}.py not found on the modelscope hub.' if dataset_py_script and dataset_formation == DatasetFormations.hf_compatible: - self.dataset = hf_data_loader( + self.dataset = hf_load_dataset( dataset_py_script, name=subset_name, revision=version, @@ -147,8 +153,126 @@ class OssDataLoader(BaseDataLoader): self.dataset.custom_map = self.dataset_context_config.data_meta_config.meta_type_map -class MaxComputeDataLoader(BaseDataLoader): - """Data loader for MaxCompute data source.""" +class VirgoDownloader(BaseDownloader): + """Data downloader for Virgo data source.""" + + def __init__(self, dataset_context_config: DatasetContextConfig): + super().__init__(dataset_context_config) + self.dataset = None + + def process(self): + """ + Sequential data fetching virgo dataset process: authorize -> build -> prepare_and_download -> post_process + """ + self._authorize() + self._build() + self._prepare_and_download() + self._post_process() + + def _authorize(self): + """Authorization of virgo dataset.""" + from modelscope.msdatasets.auth.auth_config import VirgoAuthConfig + + cookies = ModelScopeConfig.get_cookies() + user_info = ModelScopeConfig.get_user_info() + + if not self.dataset_context_config.auth_config: + auth_config = VirgoAuthConfig( + cookies=cookies, git_token='', user_info=user_info) + else: + auth_config = self.dataset_context_config.auth_config + auth_config.cookies = cookies + auth_config.git_token = '' + auth_config.user_info = user_info + + self.dataset_context_config.auth_config = auth_config + + def _build(self): + """ + Fetch virgo meta and build virgo dataset. + """ + from modelscope.msdatasets.dataset_cls import VirgoDataset + + meta_manager = DataMetaManager(self.dataset_context_config) + meta_manager.fetch_virgo_meta() + self.dataset_context_config = meta_manager.dataset_context_config + self.dataset = VirgoDataset(**self.dataset_context_config.ext_config) + + virgo_cache_dir = os.path.join( + self.dataset_context_config.cache_root_dir, + self.dataset_context_config.namespace, + self.dataset_context_config.dataset_name, + self.dataset_context_config.version) + os.makedirs( + os.path.join(virgo_cache_dir, DatasetPathName.META_NAME), + exist_ok=True) + meta_content_cache_file = os.path.join(virgo_cache_dir, + DatasetPathName.META_NAME, + 'meta_content.csv') + meta_content_df = self.dataset.meta + meta_content_df.to_csv(meta_content_cache_file, index=False) + self.dataset.meta_content_cache_file = meta_content_cache_file + self.dataset.virgo_cache_dir = virgo_cache_dir + logger.info(f'Virgo meta content saved to {meta_content_cache_file}') + + def _prepare_and_download(self): + """ + Fetch data-files from oss-urls in the virgo meta content. + """ + + if self.dataset_context_config.download_virgo_files: + import requests + import json + import shutil + from urllib.parse import urlparse + + def download_file(meta_info_val): + file_url = '' + file_path = '' + try: + file_url = json.loads(meta_info_val)['url'] + is_url = valid_url(file_url) + if is_url: + url_parse_res = urlparse(file_url) + file_name = os.path.basename(url_parse_res.path) + else: + raise ValueError(f'Unsupported url: {file_url}') + file_path = os.path.join(data_files_dir, file_name) + except Exception as e: + logger.warning(e) + + if file_path and not os.path.exists(file_path): + logger.info( + f'Downloading file from {file_url} to {file_path}') + with open(file_path, 'wb') as f: + f.write(requests.get(file_url).content) + + return file_path + + self.dataset.download_virgo_files = True + download_mode = self.dataset_context_config.download_mode + data_files_dir = os.path.join(self.dataset.virgo_cache_dir, + DatasetPathName.DATA_FILES_NAME) + if download_mode == DownloadMode.REUSE_DATASET_IF_EXISTS: + self.dataset.meta[VirgoDatasetConfig. + col_cache_file] = self.dataset.meta.apply( + lambda row: download_file(row.meta_info), + axis=1) + elif download_mode == DownloadMode.FORCE_REDOWNLOAD: + shutil.rmtree(data_files_dir, ignore_errors=True) + self.dataset.meta[VirgoDatasetConfig. + col_cache_file] = self.dataset.meta.apply( + lambda row: download_file(row.meta_info), + axis=1) + else: + raise ValueError(f'Unsupported download mode: {download_mode}') + + def _post_process(self): + ... + + +class MaxComputeDownloader(BaseDownloader): + """Data downloader for MaxCompute data source.""" # TODO: MaxCompute data source to be supported . def __init__(self, dataset_context_config: DatasetContextConfig): diff --git a/modelscope/msdatasets/data_loader/data_loader_manager.py b/modelscope/msdatasets/data_loader/data_loader_manager.py index 3c8a638a..5be32de1 100644 --- a/modelscope/msdatasets/data_loader/data_loader_manager.py +++ b/modelscope/msdatasets/data_loader/data_loader_manager.py @@ -9,7 +9,7 @@ from datasets import load_dataset as hf_data_loader from modelscope.hub.api import HubApi from modelscope.msdatasets.context.dataset_context_config import \ DatasetContextConfig -from modelscope.msdatasets.data_loader.data_loader import OssDataLoader +from modelscope.msdatasets.data_loader.data_loader import OssDownloader from modelscope.utils.constant import EXTENSIONS_TO_LOAD from modelscope.utils.logger import get_logger @@ -127,7 +127,7 @@ class RemoteDataLoaderManager(DataLoaderManager): return dataset_ret # To use the modelscope data loader elif data_loader_type == RemoteDataLoaderType.MS_DATA_LOADER: - oss_data_loader = OssDataLoader( + oss_data_loader = OssDownloader( dataset_context_config=self.dataset_context_config) oss_data_loader.process() # download statistics diff --git a/modelscope/msdatasets/dataset_cls/__init__.py b/modelscope/msdatasets/dataset_cls/__init__.py index a5b2e73d..ced94f2f 100644 --- a/modelscope/msdatasets/dataset_cls/__init__.py +++ b/modelscope/msdatasets/dataset_cls/__init__.py @@ -1,3 +1,3 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -from .dataset import ExternalDataset, NativeIterableDataset +from .dataset import ExternalDataset, NativeIterableDataset, VirgoDataset diff --git a/modelscope/msdatasets/dataset_cls/dataset.py b/modelscope/msdatasets/dataset_cls/dataset.py index 4acf51b1..e5e9664a 100644 --- a/modelscope/msdatasets/dataset_cls/dataset.py +++ b/modelscope/msdatasets/dataset_cls/dataset.py @@ -4,11 +4,13 @@ import copy import os import datasets +import pandas as pd from datasets import IterableDataset from PIL import Image -from modelscope.utils.constant import EXTENSIONS_TO_LOAD +from modelscope.utils.constant import EXTENSIONS_TO_LOAD, VirgoDatasetConfig from modelscope.utils.logger import get_logger +from modelscope.utils.url_utils import fetch_csv_with_url, valid_url logger = get_logger() @@ -108,3 +110,80 @@ class NativeIterableDataset(IterableDataset): def __len__(self): return 1 + + +class VirgoDataset(object): + """Dataset class for Virgo. + + Attributes: + _meta_content (str): Virgo meta data content, could be a url that contains csv file. + _data_type (int): Virgo dataset type, 0-Standard virgo dataset; Others-User define dataset (to be supported) + + Examples: + >>> from modelscope.msdatasets.dataset_cls import VirgoDataset + >>> input_kwargs = {'metaContent': 'http://xxx-xxx/xxx.csv', 'samplingType': 0} + >>> virgo_dataset = VirgoDataset(**input_kwargs) + >>> print(virgo_dataset[1]) + >>> print(len(virgo_dataset)) + >>> for line in virgo_dataset: + >>> print(line) + + Note: If you set `download_virgo_files` to True by using + MsDataset.load(dataset_name='your-virgo-dataset-id', hub=Hubs.virgo, download_virgo_files=True), + you can get the cache file path of the virgo dataset, the column name is `cache_file`. + >>> if virgo_dataset.download_virgo_files: + >>> print(virgo_dataset[1].get('cache_file')) + """ + + def __init__(self, **kwargs): + + self._meta_content: str = '' + self._data_type: int = 0 + self._meta: pd.DataFrame = pd.DataFrame() + + if VirgoDatasetConfig.meta_content in kwargs: + self._meta_content = kwargs.pop(VirgoDatasetConfig.meta_content) + if VirgoDatasetConfig.sampling_type in kwargs: + self._data_type = kwargs.pop(VirgoDatasetConfig.sampling_type) + + self._check_variables() + self._parse_meta() + + self.meta_content_cache_file = '' + self.virgo_cache_dir = '' + self.download_virgo_files: bool = False + + def __getitem__(self, index): + return self._meta.iloc[index].to_dict() + + def __len__(self): + return len(self._meta) + + def __iter__(self): + for _, row in self._meta.iterrows(): + yield row.to_dict() + + @property + def meta(self) -> pd.DataFrame: + """ + Virgo meta data. Contains columns: id, meta_info, analysis_result, external_info and + cache_file (if download_virgo_files is True). + """ + return self._meta + + def _parse_meta(self): + # Fetch csv content + meta_content_df = fetch_csv_with_url(self._meta_content) + self._meta = meta_content_df + + def _check_variables(self): + """Check member variables in this class. + 1. Condition-1: self._meta_content cannot be empty + 2. Condition-2: self._meta_content must be url when self._data_type is 0 + """ + if not self._meta_content: + raise 'Them meta content cannot be empty.' + if self._data_type != 0: + raise 'Supported samplingType should be 0, others are not supported yet.' + if not valid_url(self._meta_content): + raise 'The meta content must be url when data type is 0.' diff --git a/modelscope/msdatasets/meta/data_meta_manager.py b/modelscope/msdatasets/meta/data_meta_manager.py index d90b8d5e..6a34d5d6 100644 --- a/modelscope/msdatasets/meta/data_meta_manager.py +++ b/modelscope/msdatasets/meta/data_meta_manager.py @@ -140,6 +140,14 @@ class DataMetaManager(object): self.dataset_context_config.data_meta_config = data_meta_config + def fetch_virgo_meta(self) -> None: + virgo_dataset_id = self.dataset_context_config.dataset_name + version = int(self.dataset_context_config.version) + + meta_content = self.api.get_virgo_meta( + dataset_id=virgo_dataset_id, version=version) + self.dataset_context_config.ext_config = meta_content + def _fetch_meta_from_cache(self, meta_cache_dir): local_paths = defaultdict(list) dataset_type = None diff --git a/modelscope/msdatasets/ms_dataset.py b/modelscope/msdatasets/ms_dataset.py index 0a88eb91..6729e4e6 100644 --- a/modelscope/msdatasets/ms_dataset.py +++ b/modelscope/msdatasets/ms_dataset.py @@ -13,13 +13,15 @@ from datasets.utils.file_utils import is_relative_path from modelscope.hub.repository import DatasetRepository from modelscope.msdatasets.context.dataset_context_config import \ DatasetContextConfig +from modelscope.msdatasets.data_loader.data_loader import VirgoDownloader from modelscope.msdatasets.data_loader.data_loader_manager import ( LocalDataLoaderManager, LocalDataLoaderType, RemoteDataLoaderManager, RemoteDataLoaderType) +from modelscope.msdatasets.dataset_cls import (ExternalDataset, + NativeIterableDataset, + VirgoDataset) from modelscope.msdatasets.dataset_cls.custom_datasets.builder import \ build_custom_dataset -from modelscope.msdatasets.dataset_cls.dataset import (ExternalDataset, - NativeIterableDataset) from modelscope.msdatasets.utils.delete_utils import DatasetDeleteManager from modelscope.msdatasets.utils.upload_utils import DatasetUploadManager from modelscope.preprocessors import build_preprocessor @@ -28,7 +30,7 @@ from modelscope.utils.config_ds import MS_DATASETS_CACHE from modelscope.utils.constant import (DEFAULT_DATASET_NAMESPACE, DEFAULT_DATASET_REVISION, ConfigFields, DownloadMode, Hubs, ModeKeys, Tasks, - UploadMode) + UploadMode, VirgoDatasetConfig) from modelscope.utils.import_utils import is_tf_available, is_torch_available from modelscope.utils.logger import get_logger @@ -170,7 +172,7 @@ class MsDataset: use_streaming: Optional[bool] = False, custom_cfg: Optional[Config] = Config(), **config_kwargs, - ) -> Union[dict, 'MsDataset', NativeIterableDataset]: + ) -> Union[dict, 'MsDataset', NativeIterableDataset, VirgoDataset]: """Load a MsDataset from the ModelScope Hub, Hugging Face Hub, urls, or a local dataset. Args: @@ -287,6 +289,26 @@ class MsDataset: custom_cfg=custom_cfg, **config_kwargs) dataset_inst.is_custom = True return dataset_inst + elif hub == Hubs.virgo: + # Rewrite the namespace, version and cache_dir for virgo dataset. + if namespace == DEFAULT_DATASET_NAMESPACE: + dataset_context_config.namespace = VirgoDatasetConfig.default_virgo_namespace + if version == DEFAULT_DATASET_REVISION: + dataset_context_config.version = VirgoDatasetConfig.default_dataset_version + if cache_dir == MS_DATASETS_CACHE: + from modelscope.utils.config_ds import CACHE_HOME + cache_dir = os.path.join(CACHE_HOME, 'virgo', 'hub', + 'datasets') + dataset_context_config.cache_root_dir = cache_dir + if 'download_virgo_files' in config_kwargs: + dataset_context_config.download_virgo_files = config_kwargs.pop( + 'download_virgo_files') + + virgo_downloader = VirgoDownloader(dataset_context_config) + virgo_downloader.process() + + return virgo_downloader.dataset + else: raise 'Please adjust input args to specify a loading mode, we support following scenes: ' \ 'loading from local disk, huggingface hub and modelscope hub.' diff --git a/modelscope/utils/constant.py b/modelscope/utils/constant.py index 2382825a..a19eef20 100644 --- a/modelscope/utils/constant.py +++ b/modelscope/utils/constant.py @@ -327,6 +327,7 @@ class Hubs(enum.Enum): """ modelscope = 'modelscope' huggingface = 'huggingface' + virgo = 'virgo' class DownloadMode(enum.Enum): @@ -539,3 +540,23 @@ class DistributedParallelType(object): class DatasetTensorflowConfig: BATCH_SIZE = 'batch_size' DEFAULT_BATCH_SIZE_VALUE = 5 + + +class VirgoDatasetConfig: + + default_virgo_namespace = 'default_namespace' + + default_dataset_version = '1' + + env_virgo_endpoint = 'VIRGO_ENDPOINT' + + # Columns for meta request + meta_content = 'metaContent' + sampling_type = 'samplingType' + + # Columns for meta content + col_id = 'id' + col_meta_info = 'meta_info' + col_analysis_result = 'analysis_result' + col_external_info = 'external_info' + col_cache_file = 'cache_file' diff --git a/modelscope/utils/service_utils.py b/modelscope/utils/service_utils.py index 6e7c0fc1..e20988f7 100644 --- a/modelscope/utils/service_utils.py +++ b/modelscope/utils/service_utils.py @@ -8,6 +8,7 @@ import requests from modelscope.outputs import TASK_OUTPUTS, OutputKeys from modelscope.pipeline_inputs import TASK_INPUTS, InputType +from modelscope.utils.url_utils import valid_url # service data decoder func decodes data from network and convert it to pipeline's input @@ -82,12 +83,16 @@ def get_mimetype(filename): def decode_base64_to_binary(encoding): + if valid_url(encoding): + return encoding, '' extension = get_extension(encoding) data = encoding.split(',')[1] return base64.b64decode(data), extension def decode_base64_to_image(encoding): + if valid_url(encoding): + return encoding from PIL import Image content = encoding.split(';')[1] image_encoded = content.split(',')[1] diff --git a/modelscope/utils/url_utils.py b/modelscope/utils/url_utils.py new file mode 100644 index 00000000..59cc2efd --- /dev/null +++ b/modelscope/utils/url_utils.py @@ -0,0 +1,36 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from urllib.parse import urlparse + +import pandas as pd + +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +def valid_url(url) -> bool: + try: + result = urlparse(url) + return all([result.scheme, result.netloc]) + except ValueError as e: + logger.warning(e) + return False + + +def fetch_csv_with_url(csv_url: str) -> pd.DataFrame: + """Fetch the csv content from url. + + Args: + csv_url (str): The input url of csv data. + + Returns: + A pandas DataFrame object which contains the csv content. + """ + try: + df = pd.read_csv(csv_url) + except Exception as e: + logger.error(f'Failed to fetch csv from url: {csv_url}') + raise e + + return df diff --git a/tests/msdatasets/test_virgo_dataset.py b/tests/msdatasets/test_virgo_dataset.py new file mode 100644 index 00000000..03e79593 --- /dev/null +++ b/tests/msdatasets/test_virgo_dataset.py @@ -0,0 +1,76 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +import unittest + +from modelscope.hub.api import HubApi +from modelscope.msdatasets import MsDataset +from modelscope.msdatasets.dataset_cls import VirgoDataset +from modelscope.utils.constant import DownloadMode, Hubs, VirgoDatasetConfig +from modelscope.utils.logger import get_logger + +logger = get_logger() + +# Please use your own access token for buc account. +YOUR_ACCESS_TOKEN = 'your_access_token' +# Please use your own virgo dataset id and ensure you have access to it. +VIRGO_DATASET_ID = 'your_virgo_dataset_id' + + +class TestVirgoDataset(unittest.TestCase): + + def setUp(self): + self.api = HubApi() + self.api.login(YOUR_ACCESS_TOKEN) + + @unittest.skip('to be used for local test only') + def test_download_virgo_dataset_meta(self): + ds = MsDataset.load(dataset_name=VIRGO_DATASET_ID, hub=Hubs.virgo) + ds_one = next(iter(ds)) + logger.info(ds_one) + + self.assertTrue(ds_one) + self.assertIsInstance(ds_one, VirgoDataset) + self.assertIn(VirgoDatasetConfig.col_id, ds_one) + self.assertIn(VirgoDatasetConfig.col_meta_info, ds_one) + self.assertIn(VirgoDatasetConfig.col_analysis_result, ds_one) + self.assertIn(VirgoDatasetConfig.col_external_info, ds_one) + + @unittest.skip('to be used for local test only') + def test_download_virgo_dataset_files(self): + ds = MsDataset.load( + dataset_name=VIRGO_DATASET_ID, + hub=Hubs.virgo, + download_virgo_files=True) + + ds_one = next(iter(ds)) + logger.info(ds_one) + + self.assertTrue(ds_one) + self.assertIsInstance(ds_one, VirgoDataset) + self.assertTrue(ds_one.download_virgo_files) + self.assertIn(VirgoDatasetConfig.col_cache_file, ds_one) + cache_file_path = ds_one[VirgoDatasetConfig.col_cache_file] + self.assertTrue(os.path.exists(cache_file_path)) + + @unittest.skip('to be used for local test only') + def test_force_download_virgo_dataset_files(self): + ds = MsDataset.load( + dataset_name=VIRGO_DATASET_ID, + hub=Hubs.virgo, + download_mode=DownloadMode.FORCE_REDOWNLOAD, + download_virgo_files=True) + + ds_one = next(iter(ds)) + logger.info(ds_one) + + self.assertTrue(ds_one) + self.assertIsInstance(ds_one, VirgoDataset) + self.assertTrue(ds_one.download_virgo_files) + self.assertIn(VirgoDatasetConfig.col_cache_file, ds_one) + cache_file_path = ds_one[VirgoDatasetConfig.col_cache_file] + self.assertTrue(os.path.exists(cache_file_path)) + + +if __name__ == '__main__': + unittest.main() From 8ea82e6f6d58421d878ff7561bf7b57b6f6722a6 Mon Sep 17 00:00:00 2001 From: "mulin.lyh" Date: Tue, 18 Apr 2023 20:32:11 +0800 Subject: [PATCH 05/76] * add lfs append --- modelscope/hub/git.py | 21 +++++++++------------ modelscope/hub/repository.py | 21 ++++++++++++++++++++- tests/cli/test_download_cmd.py | 1 - tests/hub/test_hub_operation.py | 1 - tests/hub/test_hub_private_files.py | 1 - tests/hub/test_hub_private_repository.py | 1 - tests/hub/test_hub_repository.py | 15 ++++++++++++++- tests/hub/test_hub_retry.py | 1 - tests/hub/test_hub_revision.py | 1 - tests/hub/test_hub_revision_release_mode.py | 1 - tests/hub/test_hub_upload.py | 1 - 11 files changed, 43 insertions(+), 22 deletions(-) diff --git a/modelscope/hub/git.py b/modelscope/hub/git.py index 80887738..b0fae148 100644 --- a/modelscope/hub/git.py +++ b/modelscope/hub/git.py @@ -55,16 +55,10 @@ class GitCommandWrapper(metaclass=Singleton): response.check_returncode() return response except subprocess.CalledProcessError as error: - if response.returncode == 1: - logger.info('Nothing to commit.') - return response - else: - logger.error( - 'There are error run git command, you may need to login first.' - ) - raise GitError('stdout: %s, stderr: %s' % - (response.stdout.decode('utf8'), - error.stderr.decode('utf8'))) + logger.error('There are error run git command.') + raise GitError( + 'stdout: %s, stderr: %s' % + (response.stdout.decode('utf8'), error.stderr.decode('utf8'))) def config_auth_token(self, repo_dir, auth_token): url = self.get_repo_remote_url(repo_dir) @@ -199,8 +193,11 @@ class GitCommandWrapper(metaclass=Singleton): else: return ['/'.join(line.split('/')[1:]) for line in info[1:]] - def pull(self, repo_dir: str): - cmds = ['-C', repo_dir, 'pull'] + def pull(self, + repo_dir: str, + remote: str = 'origin', + branch: str = 'master'): + cmds = ['-C', repo_dir, 'pull', remote, branch] return self._run_git_command(*cmds) def push(self, diff --git a/modelscope/hub/repository.py b/modelscope/hub/repository.py index 1d107a3c..3fc6da2b 100644 --- a/modelscope/hub/repository.py +++ b/modelscope/hub/repository.py @@ -88,6 +88,26 @@ class Repository: remote = None return remote + def pull(self, remote: str = 'origin', branch: str = 'master'): + """Pull remote branch + + Args: + remote (str, optional): The remote name. Defaults to 'origin'. + branch (str, optional): The remote branch. Defaults to 'master'. + """ + self.git_wrapper.pull(self.model_dir, remote=remote, branch=branch) + + def add_lfs_type(self, file_name_suffix: str): + """Add file suffix to lfs list. + + Args: + file_name_suffix (str): The file name suffix. + examples '*.safetensors' + """ + os.system( + "printf '%s filter=lfs diff=lfs merge=lfs -text\n'>>%s" % + (file_name_suffix, os.path.join(self.model_dir, '.gitattributes'))) + def push(self, commit_message: str, local_branch: Optional[str] = DEFAULT_REPOSITORY_REVISION, @@ -120,7 +140,6 @@ class Repository: self.model_repo_name) url = self.git_wrapper.get_repo_remote_url(self.model_dir) - self.git_wrapper.pull(self.model_dir) self.git_wrapper.add(self.model_dir, all_files=True) self.git_wrapper.commit(self.model_dir, commit_message) diff --git a/tests/cli/test_download_cmd.py b/tests/cli/test_download_cmd.py index 53cfdadd..6059fa12 100644 --- a/tests/cli/test_download_cmd.py +++ b/tests/cli/test_download_cmd.py @@ -17,7 +17,6 @@ DEFAULT_GIT_PATH = 'git' download_model_file_name = 'test.bin' -@unittest.skip('temporarily skip') class DownloadCMDTest(unittest.TestCase): def setUp(self): diff --git a/tests/hub/test_hub_operation.py b/tests/hub/test_hub_operation.py index 82752869..a22aaa64 100644 --- a/tests/hub/test_hub_operation.py +++ b/tests/hub/test_hub_operation.py @@ -22,7 +22,6 @@ DEFAULT_GIT_PATH = 'git' download_model_file_name = 'test.bin' -@unittest.skip('temporarily skip') class HubOperationTest(unittest.TestCase): def setUp(self): diff --git a/tests/hub/test_hub_private_files.py b/tests/hub/test_hub_private_files.py index b79c11cd..a343808f 100644 --- a/tests/hub/test_hub_private_files.py +++ b/tests/hub/test_hub_private_files.py @@ -21,7 +21,6 @@ from modelscope.utils.test_utils import (TEST_ACCESS_TOKEN1, download_model_file_name = 'test.bin' -@unittest.skip('temporarily skip') class HubPrivateFileDownloadTest(unittest.TestCase): def setUp(self): diff --git a/tests/hub/test_hub_private_repository.py b/tests/hub/test_hub_private_repository.py index bd2984cf..1b7c41cd 100644 --- a/tests/hub/test_hub_private_repository.py +++ b/tests/hub/test_hub_private_repository.py @@ -17,7 +17,6 @@ from modelscope.utils.test_utils import (TEST_ACCESS_TOKEN1, DEFAULT_GIT_PATH = 'git' -@unittest.skip('temporarily skip') class HubPrivateRepositoryTest(unittest.TestCase): def setUp(self): diff --git a/tests/hub/test_hub_repository.py b/tests/hub/test_hub_repository.py index a006d7c2..7631f5db 100644 --- a/tests/hub/test_hub_repository.py +++ b/tests/hub/test_hub_repository.py @@ -26,7 +26,6 @@ DEFAULT_GIT_PATH = 'git' download_model_file_name = 'test.bin' -@unittest.skip('temporarily skip') class HubRepositoryTest(unittest.TestCase): def setUp(self): @@ -81,6 +80,20 @@ class HubRepositoryTest(unittest.TestCase): assert lfs_file1 in lfs_files assert lfs_file2 in lfs_files + def test_add_lfs_file_type(self): + repo = Repository(self.model_dir, clone_from=self.model_id) + assert os.path.exists(os.path.join(self.model_dir, ModelFile.README)) + os.chdir(self.model_dir) + lfs_file = 'test.safetensors' + os.system("echo 'safttensor'>%s" + % os.path.join(self.model_dir, lfs_file)) + repo.add_lfs_type('*.safetensors') + repo.push('test') + # check lfs files. + git_wrapper = GitCommandWrapper() + lfs_files = git_wrapper.list_lfs_files(self.model_dir) + assert lfs_file in lfs_files + if __name__ == '__main__': unittest.main() diff --git a/tests/hub/test_hub_retry.py b/tests/hub/test_hub_retry.py index 7f47f119..e294cb68 100644 --- a/tests/hub/test_hub_retry.py +++ b/tests/hub/test_hub_retry.py @@ -12,7 +12,6 @@ from modelscope.hub.api import HubApi from modelscope.hub.file_download import http_get_file -@unittest.skip('temporarily skip') class HubOperationTest(unittest.TestCase): def setUp(self): diff --git a/tests/hub/test_hub_revision.py b/tests/hub/test_hub_revision.py index e97422ad..00d5d53d 100644 --- a/tests/hub/test_hub_revision.py +++ b/tests/hub/test_hub_revision.py @@ -23,7 +23,6 @@ download_model_file_name = 'test.bin' download_model_file_name2 = 'test2.bin' -@unittest.skip('temporarily skip') class HubRevisionTest(unittest.TestCase): def setUp(self): diff --git a/tests/hub/test_hub_revision_release_mode.py b/tests/hub/test_hub_revision_release_mode.py index 49a83371..3b8416db 100644 --- a/tests/hub/test_hub_revision_release_mode.py +++ b/tests/hub/test_hub_revision_release_mode.py @@ -26,7 +26,6 @@ download_model_file_name = 'test.bin' download_model_file_name2 = 'test2.bin' -@unittest.skip('temporarily skip') class HubRevisionTest(unittest.TestCase): def setUp(self): diff --git a/tests/hub/test_hub_upload.py b/tests/hub/test_hub_upload.py index 8e439aeb..a8b90288 100644 --- a/tests/hub/test_hub_upload.py +++ b/tests/hub/test_hub_upload.py @@ -19,7 +19,6 @@ from modelscope.utils.test_utils import (TEST_ACCESS_TOKEN1, TEST_MODEL_ORG, logger = get_logger() -@unittest.skip('temporarily skip') class HubUploadTest(unittest.TestCase): def setUp(self): From 8debc432cf85d8f72b171908e914a019cc1e6706 Mon Sep 17 00:00:00 2001 From: "guangpan.cd" Date: Wed, 19 Apr 2023 17:11:25 +0800 Subject: [PATCH 06/76] Quick fix to ChineseStableDiffusion meta tensor bug. Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12382517 * Quick fix to ChineseStableDiffusion meta tensor bug. --- .../stable_diffusion/chinese_stable_diffusion_pipeline.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/chinese_stable_diffusion_pipeline.py b/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/chinese_stable_diffusion_pipeline.py index d1627962..231b4e75 100644 --- a/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/chinese_stable_diffusion_pipeline.py +++ b/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/chinese_stable_diffusion_pipeline.py @@ -46,7 +46,9 @@ class ChineseStableDiffusionPipeline(DiffusersPipeline): torch_dtype = kwargs.get('torch_dtype', torch.float32) self.pipeline = _DiffuersChineseStableDiffusionPipeline.from_pretrained( - model, torch_dtype=torch_dtype).to(self.device) + model, torch_dtype=torch_dtype) + self.pipeline.text_encoder.pooler = None + self.pipeline.to(self.device) def forward(self, inputs: Dict[str, Any], **forward_params) -> Dict[str, Any]: From dd4299e0c858d77ecf63621fe4f76da05e0d520d Mon Sep 17 00:00:00 2001 From: "bin.xue" Date: Thu, 20 Apr 2023 22:45:07 +0800 Subject: [PATCH 07/76] [to #42322933] add DFSMN model to far field KWS Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12334627 * feat: set stream mode in constructor * fix: commit model file * doc: add more information about DFSMN * fix: seperation model evaluate returns None in speechbrain whose version < 0.5.12 --- modelscope/metainfo.py | 1 + .../models/audio/kws/farfield/fsmn_sele_v3.py | 233 ++++++++++++++++++ modelscope/models/audio/kws/farfield/model.py | 31 ++- requirements/audio/audio_signal.txt | 2 +- .../test_key_word_spotting_farfield.py | 8 + .../audio/test_kws_farfield_trainer.py | 21 ++ 6 files changed, 291 insertions(+), 5 deletions(-) create mode 100644 modelscope/models/audio/kws/farfield/fsmn_sele_v3.py diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index 60ad6d85..12d39e4d 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -177,6 +177,7 @@ class Models(object): speech_frcrn_ans_cirm_16k = 'speech_frcrn_ans_cirm_16k' speech_dfsmn_ans = 'speech_dfsmn_ans' speech_dfsmn_kws_char_farfield = 'speech_dfsmn_kws_char_farfield' + speech_dfsmn_kws_char_farfield_iot = 'speech_dfsmn_kws_char_farfield_iot' speech_kws_fsmn_char_ctc_nearfield = 'speech_kws_fsmn_char_ctc_nearfield' speech_mossformer_separation_temporal_8k = 'speech_mossformer_separation_temporal_8k' kws_kwsbp = 'kws-kwsbp' diff --git a/modelscope/models/audio/kws/farfield/fsmn_sele_v3.py b/modelscope/models/audio/kws/farfield/fsmn_sele_v3.py new file mode 100644 index 00000000..d57354d0 --- /dev/null +++ b/modelscope/models/audio/kws/farfield/fsmn_sele_v3.py @@ -0,0 +1,233 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .fsmn import AffineTransform, Fsmn, LinearTransform, RectifiedLinear +from .model_def import HEADER_BLOCK_SIZE, ActivationType, LayerType, f32ToI32 + + +class DFSMNUnit(nn.Module): + """ one multi-channel deep fsmn unit + Args: + dimin: input dimension + dimexpand: feature expansion dimension + dimout: output dimension + lorder: left ofder + rorder: right order + """ + + def __init__(self, + dimin=64, + dimexpand=128, + dimout=64, + lorder=10, + rorder=1): + super(DFSMNUnit, self).__init__() + + self.expand = AffineTransform(dimin, dimexpand) + self.shrink = LinearTransform(dimexpand, dimout) + self.fsmn = Fsmn(dimout, dimout, lorder, rorder, 1, 1) + + self.debug = False + self.dataout = None + + def forward(self, input): + """ + Args: + input: [batch, time, feature] + """ + out1 = F.relu(self.expand(input)) + out2 = self.shrink(out1) + out3 = self.fsmn(out2) + + # add skip connection for matched data + if input.shape[-1] == out3.shape[-1]: + out3 = input + out3 + if self.debug: + self.dataout = out3 + return out3 + + def print_model(self): + self.expand.printModel() + self.shrink.printModel() + self.fsmn.printModel() + + def to_kaldi_nnet(self): + re_str = self.expand.toKaldiNNet() + relu = RectifiedLinear(self.expand.linear.out_features, + self.expand.linear.out_features) + re_str += relu.toKaldiNNet() + re_str = self.shrink.toKaldiNNet() + re_str += self.fsmn.toKaldiNNet() + return re_str + + +class FSMNSeleNetV3(nn.Module): + """ Deep FSMN model with channel selection performs multi-channel kws. + Zhang, Shiliang, et al. "Deep-FSMN for large vocabulary continuous speech + recognition." 2018 IEEE International Conference on Acoustics, Speech and + Signal Processing (ICASSP). IEEE, 2018. + + Args: + input_dim: input dimension + linear_dim: fsmn input dimension + proj_dim: fsmn projection dimension + lorder: fsmn left order + rorder: fsmn right order + num_syn: output dimension + fsmn_layers: no. of fsmn units + """ + + def __init__(self, + input_dim=120, + linear_dim=128, + proj_dim=64, + lorder=10, + rorder=1, + num_syn=5, + fsmn_layers=5): + super(FSMNSeleNetV3, self).__init__() + + self.mem = [] + # the first unit, mapping input dim to proj dim + unit = DFSMNUnit(input_dim, linear_dim, proj_dim, lorder, rorder) + self.mem.append(unit) + self.add_module('mem_{:d}'.format(0), unit) + + # deep fsmn layers with skip connection + for i in range(1, fsmn_layers): + unit = DFSMNUnit(proj_dim, linear_dim, proj_dim, lorder, rorder) + self.mem.append(unit) + self.add_module('mem_{:d}'.format(i), unit) + + self.expand2 = AffineTransform(proj_dim, linear_dim) + self.decision = AffineTransform(linear_dim, num_syn) + + def forward(self, input): + # multi-channel temp space, [batch, time, channel, feature] + if torch.cuda.is_available(): + x = torch.zeros(input.shape[0], input.shape[1], input.shape[2], + self.expand2.linear.out_features).cuda() + else: + x = torch.zeros(input.shape[0], input.shape[1], input.shape[2], + self.expand2.linear.out_features) + + for n in range(input.shape[2]): + chin = input[:, :, n, :] + + for unit in self.mem: + chout = unit(chin) + chin = chout + + x[:, :, n, :] = F.relu(self.expand2(chout)) + + # perform max pooling + pool = nn.MaxPool2d((x.shape[2], 1), stride=(x.shape[2], 1)) + y = pool(x) + + # remove channel dimension + y = torch.squeeze(y, -2) + z = self.decision(y) + + return z + + def print_model(self): + for unit in self.mem: + unit.print_model() + + self.expand2.printModel() + self.decision.printModel() + + def print_header(self): + """ get DFSMN params + """ + input_dim = self.mem[0].expand.linear.in_features + linear_dim = self.mem[0].expand.linear.out_features + proj_dim = self.mem[0].shrink.linear.out_features + lorder = self.mem[0].fsmn.conv_left.kernel_size[0] + rorder = 0 + if self.mem[0].fsmn.conv_right is not None: + rorder = self.mem[0].fsmn.conv_right.kernel_size[0] + + num_syn = self.decision.linear.out_features + fsmn_layers = len(self.mem) + + # no. of output channels, 0.0 means the same as numins + numouts = 1.0 + + # + # write total header + # + header = [0.0] * HEADER_BLOCK_SIZE * 5 + # numins + header[0] = 0.0 + # numouts + header[1] = numouts + # dimins + header[2] = input_dim + # dimouts + header[3] = num_syn + # numlayers + header[4] = 4 + + # + # write each layer's header + # + hidx = 1 + + header[HEADER_BLOCK_SIZE * hidx + 0] = float( + LayerType.LAYER_DFSMN.value) + header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0 + header[HEADER_BLOCK_SIZE * hidx + 2] = input_dim + header[HEADER_BLOCK_SIZE * hidx + 3] = linear_dim + header[HEADER_BLOCK_SIZE * hidx + 4] = proj_dim + header[HEADER_BLOCK_SIZE * hidx + 5] = lorder + header[HEADER_BLOCK_SIZE * hidx + 6] = rorder + header[HEADER_BLOCK_SIZE * hidx + 7] = fsmn_layers + hidx += 1 + + header[HEADER_BLOCK_SIZE * hidx + 0] = float( + LayerType.LAYER_DENSE.value) + header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0 + header[HEADER_BLOCK_SIZE * hidx + 2] = proj_dim + header[HEADER_BLOCK_SIZE * hidx + 3] = linear_dim + header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0 + header[HEADER_BLOCK_SIZE * hidx + 5] = float( + ActivationType.ACTIVATION_RELU.value) + hidx += 1 + + header[HEADER_BLOCK_SIZE * hidx + 0] = float( + LayerType.LAYER_MAX_POOLING.value) + header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0 + header[HEADER_BLOCK_SIZE * hidx + 2] = linear_dim + hidx += 1 + + header[HEADER_BLOCK_SIZE * hidx + 0] = float( + LayerType.LAYER_DENSE.value) + header[HEADER_BLOCK_SIZE * hidx + 1] = numouts + header[HEADER_BLOCK_SIZE * hidx + 2] = linear_dim + header[HEADER_BLOCK_SIZE * hidx + 3] = num_syn + header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0 + header[HEADER_BLOCK_SIZE * hidx + 5] = float( + ActivationType.ACTIVATION_SOFTMAX.value) + + for h in header: + print(f32ToI32(h)) + + def to_kaldi_nnet(self): + re_str = '\n' + for unit in self.mem: + re_str += unit.to_kaldi_nnet() + re_str = self.expand2.toKaldiNNet() + relu = RectifiedLinear(self.expand2.linear.out_features, + self.expand2.linear.out_features) + re_str += relu.toKaldiNNet() + re_str += self.decision.toKaldiNNet() + re_str += ' %d %d\n' % (self.decision.linear.out_features, + self.decision.linear.out_features) + re_str += '\n' + re_str += '\n' + + return re_str diff --git a/modelscope/models/audio/kws/farfield/model.py b/modelscope/models/audio/kws/farfield/model.py index fff88805..670ac97c 100644 --- a/modelscope/models/audio/kws/farfield/model.py +++ b/modelscope/models/audio/kws/farfield/model.py @@ -11,6 +11,7 @@ from modelscope.models.builder import MODELS from modelscope.utils.audio.audio_utils import update_conf from modelscope.utils.constant import Tasks from .fsmn_sele_v2 import FSMNSeleNetV2 +from .fsmn_sele_v3 import FSMNSeleNetV3 @MODELS.register_module( @@ -18,6 +19,7 @@ from .fsmn_sele_v2 import FSMNSeleNetV2 class FSMNSeleNetV2Decorator(TorchModel): r""" A decorator of FSMNSeleNetV2 for integrating into modelscope framework """ + MODEL_CLASS = FSMNSeleNetV2 MODEL_TXT = 'model.txt' SC_CONFIG = 'sound_connect.conf' @@ -33,7 +35,7 @@ class FSMNSeleNetV2Decorator(TorchModel): """ super().__init__(model_dir, *args, **kwargs) if training: - self.model = FSMNSeleNetV2(*args, **kwargs) + self.model = self.MODEL_CLASS(*args, **kwargs) else: sc_config_file = os.path.join(model_dir, self.SC_CONFIG) model_txt_file = os.path.join(model_dir, self.MODEL_TXT) @@ -42,7 +44,7 @@ class FSMNSeleNetV2Decorator(TorchModel): self._sc = None if os.path.exists(model_txt_file): - conf_dict = dict(mode=56542, kws_model=model_txt_file) + conf_dict = dict(kws_model=model_txt_file) update_conf(sc_config_file, new_config_file, conf_dict) import py_sound_connect self._sc = py_sound_connect.SoundConnect(new_config_file) @@ -50,8 +52,8 @@ class FSMNSeleNetV2Decorator(TorchModel): self.size_out = self._sc.bytesPerBlockOut() else: raise Exception( - f'Invalid model directory! Failed to load model file: {model_txt_file}.' - ) + f'Invalid model directory! Failed to load model file:' + f' {model_txt_file}.') def __del__(self): if hasattr(self, 'tmp_dir'): @@ -73,3 +75,24 @@ class FSMNSeleNetV2Decorator(TorchModel): 'confidence': self._sc.kwsConfidence() } return result + + +@MODELS.register_module( + Tasks.keyword_spotting, + module_name=Models.speech_dfsmn_kws_char_farfield_iot) +class FSMNSeleNetV3Decorator(FSMNSeleNetV2Decorator): + r""" A decorator of FSMNSeleNetV3 for integrating into modelscope framework """ + + MODEL_CLASS = FSMNSeleNetV3 + + def __init__(self, + model_dir: str, + training: Optional[bool] = False, + *args, + **kwargs): + """initialize the dfsmn model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, training, *args, **kwargs) diff --git a/requirements/audio/audio_signal.txt b/requirements/audio/audio_signal.txt index 61e688f3..71037baa 100644 --- a/requirements/audio/audio_signal.txt +++ b/requirements/audio/audio_signal.txt @@ -6,6 +6,6 @@ numpy rotary_embedding_torch>=0.1.5 scipy SoundFile>0.10 -speechbrain>=0.5.7 +speechbrain>=0.5.12 torchaudio tqdm diff --git a/tests/pipelines/test_key_word_spotting_farfield.py b/tests/pipelines/test_key_word_spotting_farfield.py index e736f48b..3193149c 100644 --- a/tests/pipelines/test_key_word_spotting_farfield.py +++ b/tests/pipelines/test_key_word_spotting_farfield.py @@ -19,6 +19,7 @@ class KWSFarfieldTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/speech_dfsmn_kws_char_farfield_16k_nihaomiya' + self.model_id_iot = 'damo/speech_dfsmn_kws_char_farfield_iot_16k_nihaomiya' if os.path.isfile(OUTPUT_WAV): os.remove(OUTPUT_WAV) @@ -29,6 +30,13 @@ class KWSFarfieldTest(unittest.TestCase): self.assertEqual(len(result['kws_list']), 5) print(result['kws_list'][-1]) + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') + def test_normal_iot(self): + kws = pipeline(Tasks.keyword_spotting, model=self.model_id_iot) + result = kws(os.path.join(os.getcwd(), TEST_SPEECH_FILE)) + self.assertEqual(len(result['kws_list']), 5) + print(result['kws_list'][-1]) + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_output(self): kws = pipeline(Tasks.keyword_spotting, model=self.model_id) diff --git a/tests/trainers/audio/test_kws_farfield_trainer.py b/tests/trainers/audio/test_kws_farfield_trainer.py index cc2b38f6..9bf65e04 100644 --- a/tests/trainers/audio/test_kws_farfield_trainer.py +++ b/tests/trainers/audio/test_kws_farfield_trainer.py @@ -23,6 +23,7 @@ class TestKwsFarfieldTrainer(unittest.TestCase): if not os.path.exists(self.tmp_dir): os.makedirs(self.tmp_dir) self.model_id = 'damo/speech_dfsmn_kws_char_farfield_16k_nihaomiya' + self.model_id_iot = 'damo/speech_dfsmn_kws_char_farfield_iot_16k_nihaomiya' train_pos_list = self.create_list('pos.list', POS_FILE) train_neg_list = self.create_list('neg.list', NEG_FILE) @@ -83,3 +84,23 @@ class TestKwsFarfieldTrainer(unittest.TestCase): f'work_dir:{self.tmp_dir}') self.assertIn('val_dataset.bin', results_files, f'work_dir:{self.tmp_dir}') + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_normal_iot(self): + kwargs = dict( + model=self.model_id_iot, + work_dir=self.tmp_dir, + workers=2, + max_epochs=2, + train_iters_per_epoch=2, + val_iters_per_epoch=1, + custom_conf=self.custom_conf) + + trainer = build_trainer( + Trainers.speech_dfsmn_kws_char_farfield, default_args=kwargs) + trainer.train() + results_files = os.listdir(self.tmp_dir) + self.assertIn(f'{trainer.timestamp}.log.json', results_files, + f'work_dir:{self.tmp_dir}') + self.assertIn('val_dataset.bin', results_files, + f'work_dir:{self.tmp_dir}') From 92f7c66d12e8371a5779d025cbc3985a536ced80 Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Fri, 21 Apr 2023 13:49:17 +0800 Subject: [PATCH 08/76] Add progress bar and fix cache dir issue 1. Add progress bar for dataframe apply when downloading data files. 2. Fix cache data files issue 3. Refine _prepare_and_download() func Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12408412 * add progress bar and fix cache dir * add progress bar and fix cache dir * refine code --- .../msdatasets/data_loader/data_loader.py | 33 ++++++++++--------- tests/msdatasets/test_virgo_dataset.py | 10 +++--- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/modelscope/msdatasets/data_loader/data_loader.py b/modelscope/msdatasets/data_loader/data_loader.py index 38b83c46..ebda87e6 100644 --- a/modelscope/msdatasets/data_loader/data_loader.py +++ b/modelscope/msdatasets/data_loader/data_loader.py @@ -225,10 +225,10 @@ class VirgoDownloader(BaseDownloader): import json import shutil from urllib.parse import urlparse + from functools import partial - def download_file(meta_info_val): + def download_file(meta_info_val, data_dir): file_url = '' - file_path = '' try: file_url = json.loads(meta_info_val)['url'] is_url = valid_url(file_url) @@ -237,13 +237,15 @@ class VirgoDownloader(BaseDownloader): file_name = os.path.basename(url_parse_res.path) else: raise ValueError(f'Unsupported url: {file_url}') - file_path = os.path.join(data_files_dir, file_name) + file_path = os.path.join(data_dir, file_name) except Exception as e: - logger.warning(e) + logger.error(e) + file_path = '' if file_path and not os.path.exists(file_path): logger.info( f'Downloading file from {file_url} to {file_path}') + os.makedirs(data_dir, exist_ok=True) with open(file_path, 'wb') as f: f.write(requests.get(file_url).content) @@ -253,19 +255,18 @@ class VirgoDownloader(BaseDownloader): download_mode = self.dataset_context_config.download_mode data_files_dir = os.path.join(self.dataset.virgo_cache_dir, DatasetPathName.DATA_FILES_NAME) - if download_mode == DownloadMode.REUSE_DATASET_IF_EXISTS: - self.dataset.meta[VirgoDatasetConfig. - col_cache_file] = self.dataset.meta.apply( - lambda row: download_file(row.meta_info), - axis=1) - elif download_mode == DownloadMode.FORCE_REDOWNLOAD: + + if download_mode == DownloadMode.FORCE_REDOWNLOAD: shutil.rmtree(data_files_dir, ignore_errors=True) - self.dataset.meta[VirgoDatasetConfig. - col_cache_file] = self.dataset.meta.apply( - lambda row: download_file(row.meta_info), - axis=1) - else: - raise ValueError(f'Unsupported download mode: {download_mode}') + + from tqdm import tqdm + tqdm.pandas(desc='apply download_file') + self.dataset.meta[ + VirgoDatasetConfig. + col_cache_file] = self.dataset.meta.progress_apply( + lambda row: partial( + download_file, data_dir=data_files_dir)(row.meta_info), + axis=1) def _post_process(self): ... diff --git a/tests/msdatasets/test_virgo_dataset.py b/tests/msdatasets/test_virgo_dataset.py index 03e79593..219b7052 100644 --- a/tests/msdatasets/test_virgo_dataset.py +++ b/tests/msdatasets/test_virgo_dataset.py @@ -30,7 +30,7 @@ class TestVirgoDataset(unittest.TestCase): logger.info(ds_one) self.assertTrue(ds_one) - self.assertIsInstance(ds_one, VirgoDataset) + self.assertIsInstance(ds, VirgoDataset) self.assertIn(VirgoDatasetConfig.col_id, ds_one) self.assertIn(VirgoDatasetConfig.col_meta_info, ds_one) self.assertIn(VirgoDatasetConfig.col_analysis_result, ds_one) @@ -47,8 +47,8 @@ class TestVirgoDataset(unittest.TestCase): logger.info(ds_one) self.assertTrue(ds_one) - self.assertIsInstance(ds_one, VirgoDataset) - self.assertTrue(ds_one.download_virgo_files) + self.assertIsInstance(ds, VirgoDataset) + self.assertTrue(ds.download_virgo_files) self.assertIn(VirgoDatasetConfig.col_cache_file, ds_one) cache_file_path = ds_one[VirgoDatasetConfig.col_cache_file] self.assertTrue(os.path.exists(cache_file_path)) @@ -65,8 +65,8 @@ class TestVirgoDataset(unittest.TestCase): logger.info(ds_one) self.assertTrue(ds_one) - self.assertIsInstance(ds_one, VirgoDataset) - self.assertTrue(ds_one.download_virgo_files) + self.assertIsInstance(ds, VirgoDataset) + self.assertTrue(ds.download_virgo_files) self.assertIn(VirgoDatasetConfig.col_cache_file, ds_one) cache_file_path = ds_one[VirgoDatasetConfig.col_cache_file] self.assertTrue(os.path.exists(cache_file_path)) From b19cc13ca72565e3e04861b9a6a5a6af0a301107 Mon Sep 17 00:00:00 2001 From: "yuanzhi.zyz" Date: Fri, 21 Apr 2023 15:05:16 +0800 Subject: [PATCH 09/76] update ocr_reco MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 增加本地lmdb输入的接口； 2. finetune后模型的keys有prefix，导致不能直接读取，已在model初始化时统一替换； 3. 添加各个模型的cpu和gpu测试。 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12256669 * add local_lmdb read --- modelscope/models/cv/ocr_recognition/model.py | 11 +- .../ocr_recognition_dataset.py | 6 +- tests/pipelines/test_ocr_recognition.py | 116 +++++++++++++++++- 3 files changed, 126 insertions(+), 7 deletions(-) diff --git a/modelscope/models/cv/ocr_recognition/model.py b/modelscope/models/cv/ocr_recognition/model.py index 6eb13403..2406b6dc 100644 --- a/modelscope/models/cv/ocr_recognition/model.py +++ b/modelscope/models/cv/ocr_recognition/model.py @@ -90,8 +90,15 @@ class OCRRecognition(TorchModel): f'recognizer should be either ConvNextViT, CRNN, but got {cfgs.model.recognizer}' ) if model_path != '': - self.recognizer.load_state_dict( - torch.load(model_path, map_location='cpu')) + params_pretrained = torch.load(model_path, map_location='cpu') + model_dict = self.recognizer.state_dict() + # remove prefix for finetuned models + check_point = { + k.replace('recognizer.', ''): v + for k, v in params_pretrained.items() + } + model_dict.update(check_point) + self.recognizer.load_state_dict(model_dict) dict_path = os.path.join(model_dir, ModelFile.VOCAB_FILE) self.labelMapping = dict() diff --git a/modelscope/msdatasets/dataset_cls/custom_datasets/ocr_recognition_dataset.py b/modelscope/msdatasets/dataset_cls/custom_datasets/ocr_recognition_dataset.py index bc9cd3ca..bfbb6eb3 100644 --- a/modelscope/msdatasets/dataset_cls/custom_datasets/ocr_recognition_dataset.py +++ b/modelscope/msdatasets/dataset_cls/custom_datasets/ocr_recognition_dataset.py @@ -34,10 +34,12 @@ def Q2B(uchar): Tasks.ocr_recognition, module_name=Models.ocr_recognition) class OCRRecognitionDataset(TorchCustomDataset): - def __init__(self, **kwargs): + def __init__(self, local_lmdb=None, preprocessor=None, **kwargs): split_config = kwargs['split_config'] cache_root = next(iter(split_config.values())) lmdb_path = os.path.join(cache_root, DATASET_STRUCTURE['lmdb']) + if local_lmdb is not None: + lmdb_path = local_lmdb self.env = lmdb.open( lmdb_path, max_readers=1, @@ -51,7 +53,7 @@ class OCRRecognitionDataset(TorchCustomDataset): self.nSamples = 0 with self.env.begin(write=False) as txn: self.nSamples = int(txn.get('num-samples'.encode())) - self.reco_preprocess = kwargs['preprocessor'] + self.reco_preprocess = preprocessor def __len__(self): return self.nSamples diff --git a/tests/pipelines/test_ocr_recognition.py b/tests/pipelines/test_ocr_recognition.py index 145ae22a..3953ae63 100644 --- a/tests/pipelines/test_ocr_recognition.py +++ b/tests/pipelines/test_ocr_recognition.py @@ -13,7 +13,7 @@ from modelscope.utils.test_utils import test_level class OCRRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): def setUp(self) -> None: - self.model_id = 'damo/cv_crnn_ocr-recognition-general_damo' + self.model_id = 'damo/cv_convnextTiny_ocr-recognition-general_damo' self.test_image = 'data/test/images/ocr_recognition.jpg' self.task = Tasks.ocr_recognition @@ -26,7 +26,47 @@ class OCRRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): ocr_recognition = pipeline( Tasks.ocr_recognition, model=self.model_id, - model_revision='v2.2.1') + model_revision='v2.3.0') + self.pipeline_inference(ocr_recognition, self.test_image) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_from_modelhub_handwritten(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, + model='damo/cv_convnextTiny_ocr-recognition-handwritten_damo', + model_revision='v2.3.0') + self.pipeline_inference(ocr_recognition, self.test_image) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_from_modelhub_scene(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, + model='damo/cv_convnextTiny_ocr-recognition-scene_damo', + model_revision='v2.3.0') + self.pipeline_inference(ocr_recognition, self.test_image) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_from_modelhub_document(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, + model='damo/cv_convnextTiny_ocr-recognition-document_damo', + model_revision='v2.3.0') + self.pipeline_inference(ocr_recognition, self.test_image) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_from_modelhub_licenseplate(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, + model='damo/cv_convnextTiny_ocr-recognition-licenseplate_damo', + model_revision='v2.3.0') + self.pipeline_inference(ocr_recognition, self.test_image) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_from_modelhub_crnn(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, + model='damo/cv_crnn_ocr-recognition-general_damo', + model_revision='v2.2.2') self.pipeline_inference(ocr_recognition, self.test_image) @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') @@ -34,7 +74,7 @@ class OCRRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): ocr_recognition = pipeline( Tasks.ocr_recognition, model=self.model_id, - model_revision='v2.2.1') + model_revision='v2.3.0') imagePIL = PIL.Image.open(self.test_image) self.pipeline_inference(ocr_recognition, imagePIL) @@ -44,6 +84,76 @@ class OCRRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): Tasks.ocr_recognition, model_revision='v2.3.0') self.pipeline_inference(ocr_recognition, self.test_image) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_from_modelhub_cpu(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, + model=self.model_id, + model_revision='v2.3.0', + device='cpu') + self.pipeline_inference(ocr_recognition, self.test_image) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_from_modelhub_handwritten_cpu(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, + model='damo/cv_convnextTiny_ocr-recognition-handwritten_damo', + model_revision='v2.3.0', + device='cpu') + self.pipeline_inference(ocr_recognition, self.test_image) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_from_modelhub_scene_cpu(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, + model='damo/cv_convnextTiny_ocr-recognition-scene_damo', + model_revision='v2.3.0', + device='cpu') + self.pipeline_inference(ocr_recognition, self.test_image) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_from_modelhub_document_cpu(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, + model='damo/cv_convnextTiny_ocr-recognition-document_damo', + model_revision='v2.3.0', + device='cpu') + self.pipeline_inference(ocr_recognition, self.test_image) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_from_modelhub_licenseplate_cpu(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, + model='damo/cv_convnextTiny_ocr-recognition-licenseplate_damo', + model_revision='v2.3.0', + device='cpu') + self.pipeline_inference(ocr_recognition, self.test_image) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_from_modelhub_crnn_cpu(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, + model='damo/cv_crnn_ocr-recognition-general_damo', + model_revision='v2.2.2', + device='cpu') + self.pipeline_inference(ocr_recognition, self.test_image) + + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') + def test_run_with_model_from_modelhub_PILinput_cpu(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, + model=self.model_id, + model_revision='v2.3.0', + device='cpu') + imagePIL = PIL.Image.open(self.test_image) + self.pipeline_inference(ocr_recognition, imagePIL) + + @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') + def test_run_modelhub_default_model_cpu(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, model_revision='v2.3.0', device='cpu') + self.pipeline_inference(ocr_recognition, self.test_image) + @unittest.skip('demo compatibility test is only enabled on a needed-basis') def test_demo_compatibility(self): self.compatibility_check() From 1719ecad03786fca1e20dd6fb549671a7f615119 Mon Sep 17 00:00:00 2001 From: "zhangzhicheng.zzc" Date: Fri, 21 Apr 2023 18:34:29 +0800 Subject: [PATCH 10/76] stable diffusion allow postprocess kwargs Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12418063 * stable diffusion allow postprocess kwargs --- .../multi_modal/diffusers_wrapped/diffusers_pipeline.py | 4 ++-- .../stable_diffusion/chinese_stable_diffusion_pipeline.py | 2 +- .../stable_diffusion/stable_diffusion_pipeline.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modelscope/pipelines/multi_modal/diffusers_wrapped/diffusers_pipeline.py b/modelscope/pipelines/multi_modal/diffusers_wrapped/diffusers_pipeline.py index d1e3a2ae..ce0455b6 100644 --- a/modelscope/pipelines/multi_modal/diffusers_wrapped/diffusers_pipeline.py +++ b/modelscope/pipelines/multi_modal/diffusers_wrapped/diffusers_pipeline.py @@ -39,10 +39,10 @@ class DiffusersPipeline(Pipeline): self.models = [self.model] self.has_multiple_models = len(self.models) > 1 - def preprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + def preprocess(self, inputs: Dict[str, Any], **kwargs) -> Dict[str, Any]: return inputs - def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + def postprocess(self, inputs: Dict[str, Any], **kwargs) -> Dict[str, Any]: return inputs def __call__(self, input: Union[Input, List[Input]], *args, diff --git a/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/chinese_stable_diffusion_pipeline.py b/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/chinese_stable_diffusion_pipeline.py index 231b4e75..539fd4ba 100644 --- a/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/chinese_stable_diffusion_pipeline.py +++ b/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/chinese_stable_diffusion_pipeline.py @@ -75,7 +75,7 @@ class ChineseStableDiffusionPipeline(DiffusersPipeline): callback=inputs.get('callback'), callback_steps=inputs.get('callback_steps', 1)) - def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + def postprocess(self, inputs: Dict[str, Any], **kwargs) -> Dict[str, Any]: images = [] for img in inputs.images: if isinstance(img, Image.Image): diff --git a/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py b/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py index b6d9d3bd..49b4ef37 100644 --- a/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py +++ b/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py @@ -65,7 +65,7 @@ class StableDiffusionWrapperPipeline(DiffusersPipeline): callback=inputs.get('callback'), callback_steps=inputs.get('callback_steps', 1)) - def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + def postprocess(self, inputs: Dict[str, Any], **kwargs) -> Dict[str, Any]: images = [] for img in inputs.images: if isinstance(img, Image.Image): From 93f73a26e757c6302ed05acb7d9349b61c59d97f Mon Sep 17 00:00:00 2001 From: "chenyafeng.cyf" Date: Wed, 26 Apr 2023 11:18:28 +0800 Subject: [PATCH 11/76] add rdino model Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12406691 --- .gitignore | 3 + modelscope/metainfo.py | 2 + modelscope/models/audio/sv/rdino.py | 573 ++++++++++++++++++ .../speaker_verification_rdino_pipeline.py | 110 ++++ tests/pipelines/test_speaker_verification.py | 19 +- 5 files changed, 705 insertions(+), 2 deletions(-) create mode 100644 modelscope/models/audio/sv/rdino.py create mode 100644 modelscope/pipelines/audio/speaker_verification_rdino_pipeline.py diff --git a/.gitignore b/.gitignore index 790daab3..6cc2df63 100644 --- a/.gitignore +++ b/.gitignore @@ -128,3 +128,6 @@ result.mp4 # Pytorch *.pth *.pt + +# ast template +ast_index_file.py diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index 12d39e4d..432f7dd8 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -188,6 +188,7 @@ class Models(object): generic_sv = 'generic-sv' ecapa_tdnn_sv = 'ecapa-tdnn-sv' campplus_sv = 'cam++-sv' + rdino_tdnn_sv = 'rdino_ecapa-tdnn-sv' generic_lm = 'generic-lm' # multi-modal models @@ -486,6 +487,7 @@ class Pipelines(object): speaker_diarization_inference = 'speaker-diarization-inference' vad_inference = 'vad-inference' speaker_verification = 'speaker-verification' + speaker_verification_rdino = 'speaker-verification-rdino' lm_inference = 'language-score-prediction' speech_timestamp_inference = 'speech-timestamp-inference' diff --git a/modelscope/models/audio/sv/rdino.py b/modelscope/models/audio/sv/rdino.py new file mode 100644 index 00000000..0d51ee7a --- /dev/null +++ b/modelscope/models/audio/sv/rdino.py @@ -0,0 +1,573 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +""" This ECAPA-TDNN implementation is adapted from https://github.com/speechbrain/speechbrain. + RDINOHead implementation is adapted from DINO framework. +""" +import math +import os +from typing import Any, Dict, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchaudio.compliance.kaldi as Kaldi + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.utils.constant import Tasks + + +def length_to_mask(length, max_len=None, dtype=None, device=None): + assert len(length.shape) == 1 + + if max_len is None: + max_len = length.max().long().item() + mask = torch.arange( + max_len, device=length.device, dtype=length.dtype).expand( + len(length), max_len) < length.unsqueeze(1) + + if dtype is None: + dtype = length.dtype + + if device is None: + device = length.device + + mask = torch.as_tensor(mask, dtype=dtype, device=device) + return mask + + +def get_padding_elem(L_in: int, stride: int, kernel_size: int, dilation: int): + if stride > 1: + n_steps = math.ceil(((L_in - kernel_size * dilation) / stride) + 1) + L_out = stride * (n_steps - 1) + kernel_size * dilation + padding = [kernel_size // 2, kernel_size // 2] + + else: + L_out = (L_in - dilation * (kernel_size - 1) - 1) // stride + 1 + + padding = [(L_in - L_out) // 2, (L_in - L_out) // 2] + return padding + + +class Conv1d(nn.Module): + + def __init__( + self, + out_channels, + kernel_size, + in_channels, + stride=1, + dilation=1, + padding='same', + groups=1, + bias=True, + padding_mode='reflect', + ): + super().__init__() + self.kernel_size = kernel_size + self.stride = stride + self.dilation = dilation + self.padding = padding + self.padding_mode = padding_mode + + self.conv = nn.Conv1d( + in_channels, + out_channels, + self.kernel_size, + stride=self.stride, + dilation=self.dilation, + padding=0, + groups=groups, + bias=bias, + ) + + def forward(self, x): + if self.padding == 'same': + x = self._manage_padding(x, self.kernel_size, self.dilation, + self.stride) + + elif self.padding == 'causal': + num_pad = (self.kernel_size - 1) * self.dilation + x = F.pad(x, (num_pad, 0)) + + elif self.padding == 'valid': + pass + + else: + raise ValueError( + "Padding must be 'same', 'valid' or 'causal'. Got " + + self.padding) + + wx = self.conv(x) + + return wx + + def _manage_padding( + self, + x, + kernel_size: int, + dilation: int, + stride: int, + ): + L_in = x.shape[-1] + padding = get_padding_elem(L_in, stride, kernel_size, dilation) + x = F.pad(x, padding, mode=self.padding_mode) + + return x + + +class BatchNorm1d(nn.Module): + + def __init__( + self, + input_size, + eps=1e-05, + momentum=0.1, + ): + super().__init__() + self.norm = nn.BatchNorm1d( + input_size, + eps=eps, + momentum=momentum, + ) + + def forward(self, x): + return self.norm(x) + + +class TDNNBlock(nn.Module): + + def __init__( + self, + in_channels, + out_channels, + kernel_size, + dilation, + activation=nn.ReLU, + groups=1, + ): + super(TDNNBlock, self).__init__() + self.conv = Conv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + dilation=dilation, + groups=groups, + ) + self.activation = activation() + self.norm = BatchNorm1d(input_size=out_channels) + + def forward(self, x): + return self.norm(self.activation(self.conv(x))) + + +class Res2NetBlock(torch.nn.Module): + + def __init__(self, + in_channels, + out_channels, + scale=8, + kernel_size=3, + dilation=1): + super(Res2NetBlock, self).__init__() + assert in_channels % scale == 0 + assert out_channels % scale == 0 + + in_channel = in_channels // scale + hidden_channel = out_channels // scale + + self.blocks = nn.ModuleList([ + TDNNBlock( + in_channel, + hidden_channel, + kernel_size=kernel_size, + dilation=dilation, + ) for i in range(scale - 1) + ]) + self.scale = scale + + def forward(self, x): + y = [] + for i, x_i in enumerate(torch.chunk(x, self.scale, dim=1)): + if i == 0: + y_i = x_i + elif i == 1: + y_i = self.blocks[i - 1](x_i) + else: + y_i = self.blocks[i - 1](x_i + y_i) + y.append(y_i) + y = torch.cat(y, dim=1) + return y + + +class SEBlock(nn.Module): + + def __init__(self, in_channels, se_channels, out_channels): + super(SEBlock, self).__init__() + + self.conv1 = Conv1d( + in_channels=in_channels, out_channels=se_channels, kernel_size=1) + self.relu = torch.nn.ReLU(inplace=True) + self.conv2 = Conv1d( + in_channels=se_channels, out_channels=out_channels, kernel_size=1) + self.sigmoid = torch.nn.Sigmoid() + + def forward(self, x, lengths=None): + L = x.shape[-1] + if lengths is not None: + mask = length_to_mask(lengths * L, max_len=L, device=x.device) + mask = mask.unsqueeze(1) + total = mask.sum(dim=2, keepdim=True) + s = (x * mask).sum(dim=2, keepdim=True) / total + else: + s = x.mean(dim=2, keepdim=True) + + s = self.relu(self.conv1(s)) + s = self.sigmoid(self.conv2(s)) + + return s * x + + +class AttentiveStatisticsPooling(nn.Module): + + def __init__(self, channels, attention_channels=128, global_context=True): + super().__init__() + + self.eps = 1e-12 + self.global_context = global_context + if global_context: + self.tdnn = TDNNBlock(channels * 3, attention_channels, 1, 1) + else: + self.tdnn = TDNNBlock(channels, attention_channels, 1, 1) + self.tanh = nn.Tanh() + self.conv = Conv1d( + in_channels=attention_channels, + out_channels=channels, + kernel_size=1) + + def forward(self, x, lengths=None): + L = x.shape[-1] + + def _compute_statistics(x, m, dim=2, eps=self.eps): + mean = (m * x).sum(dim) + std = torch.sqrt( + (m * (x - mean.unsqueeze(dim)).pow(2)).sum(dim).clamp(eps)) + return mean, std + + if lengths is None: + lengths = torch.ones(x.shape[0], device=x.device) + + # Make binary mask of shape [N, 1, L] + mask = length_to_mask(lengths * L, max_len=L, device=x.device) + mask = mask.unsqueeze(1) + + # Expand the temporal context of the pooling layer by allowing the + # self-attention to look at global properties of the utterance. + if self.global_context: + # torch.std is unstable for backward computation + # https://github.com/pytorch/pytorch/issues/4320 + total = mask.sum(dim=2, keepdim=True).float() + mean, std = _compute_statistics(x, mask / total) + mean = mean.unsqueeze(2).repeat(1, 1, L) + std = std.unsqueeze(2).repeat(1, 1, L) + attn = torch.cat([x, mean, std], dim=1) + else: + attn = x + + # Apply layers + attn = self.conv(self.tanh(self.tdnn(attn))) + + # Filter out zero-paddings + attn = attn.masked_fill(mask == 0, float('-inf')) + + attn = F.softmax(attn, dim=2) + mean, std = _compute_statistics(x, attn) + # Append mean and std of the batch + pooled_stats = torch.cat((mean, std), dim=1) + pooled_stats = pooled_stats.unsqueeze(2) + + return pooled_stats + + +class SERes2NetBlock(nn.Module): + + def __init__( + self, + in_channels, + out_channels, + res2net_scale=8, + se_channels=128, + kernel_size=1, + dilation=1, + activation=torch.nn.ReLU, + groups=1, + ): + super().__init__() + self.out_channels = out_channels + self.tdnn1 = TDNNBlock( + in_channels, + out_channels, + kernel_size=1, + dilation=1, + activation=activation, + groups=groups, + ) + self.res2net_block = Res2NetBlock(out_channels, out_channels, + res2net_scale, kernel_size, dilation) + self.tdnn2 = TDNNBlock( + out_channels, + out_channels, + kernel_size=1, + dilation=1, + activation=activation, + groups=groups, + ) + self.se_block = SEBlock(out_channels, se_channels, out_channels) + + self.shortcut = None + if in_channels != out_channels: + self.shortcut = Conv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=1, + ) + + def forward(self, x, lengths=None): + residual = x + if self.shortcut: + residual = self.shortcut(x) + + x = self.tdnn1(x) + x = self.res2net_block(x) + x = self.tdnn2(x) + x = self.se_block(x, lengths) + + return x + residual + + +class ECAPA_TDNN(nn.Module): + """An implementation of the speaker embedding model in a paper. + "ECAPA-TDNN: Emphasized Channel Attention, Propagation and Aggregation in + TDNN Based Speaker Verification" (https://arxiv.org/abs/2005.07143). + """ + + def __init__( + self, + input_size, + device='cpu', + lin_neurons=512, + activation=torch.nn.ReLU, + channels=[512, 512, 512, 512, 1536], + kernel_sizes=[5, 3, 3, 3, 1], + dilations=[1, 2, 3, 4, 1], + attention_channels=128, + res2net_scale=8, + se_channels=128, + global_context=True, + groups=[1, 1, 1, 1, 1], + ): + + super().__init__() + assert len(channels) == len(kernel_sizes) + assert len(channels) == len(dilations) + self.channels = channels + self.blocks = nn.ModuleList() + + # The initial TDNN layer + self.blocks.append( + TDNNBlock( + input_size, + channels[0], + kernel_sizes[0], + dilations[0], + activation, + groups[0], + )) + + # SE-Res2Net layers + for i in range(1, len(channels) - 1): + self.blocks.append( + SERes2NetBlock( + channels[i - 1], + channels[i], + res2net_scale=res2net_scale, + se_channels=se_channels, + kernel_size=kernel_sizes[i], + dilation=dilations[i], + activation=activation, + groups=groups[i], + )) + + # Multi-layer feature aggregation + self.mfa = TDNNBlock( + channels[-1], + channels[-1], + kernel_sizes[-1], + dilations[-1], + activation, + groups=groups[-1], + ) + + # Attentive Statistical Pooling + self.asp = AttentiveStatisticsPooling( + channels[-1], + attention_channels=attention_channels, + global_context=global_context, + ) + self.asp_bn = BatchNorm1d(input_size=channels[-1] * 2) + + # Final linear transformation + self.fc = Conv1d( + in_channels=channels[-1] * 2, + out_channels=lin_neurons, + kernel_size=1, + ) + + def forward(self, x, lengths=None): + """Returns the embedding vector. + + Arguments + --------- + x : torch.Tensor + Tensor of shape (batch, time, channel). + """ + x = x.transpose(1, 2) + + xl = [] + for layer in self.blocks: + try: + x = layer(x, lengths=lengths) + except TypeError: + x = layer(x) + xl.append(x) + + # Multi-layer feature aggregation + x = torch.cat(xl[1:], dim=1) + x = self.mfa(x) + + # Attentive Statistical Pooling + x = self.asp(x, lengths=lengths) + x = self.asp_bn(x) + + # Final linear transformation + x = self.fc(x) + + x = x.transpose(1, 2).squeeze(1) + return x + + +class RDINOHead(nn.Module): + + def __init__(self, + in_dim, + out_dim, + use_bn=False, + norm_last_layer=True, + nlayers=3, + hidden_dim=2048, + bottleneck_dim=256, + add_dim=8192): + super().__init__() + nlayers = max(nlayers, 1) + if nlayers == 1: + self.mlp = nn.Linear(in_dim, bottleneck_dim) + else: + layers = [nn.Linear(in_dim, hidden_dim)] + if use_bn: + layers.append(nn.BatchNorm1d(hidden_dim)) + layers.append(nn.GELU()) + for _ in range(nlayers - 2): + layers.append(nn.Linear(hidden_dim, hidden_dim)) + if use_bn: + layers.append(nn.BatchNorm1d(hidden_dim)) + layers.append(nn.GELU()) + + layers.append(nn.Linear(hidden_dim, add_dim)) + self.mlp = nn.Sequential(*layers) + self.add_layer = nn.Linear(add_dim, bottleneck_dim) + self.apply(self._init_weights) + self.last_layer = nn.utils.weight_norm( + nn.Linear(bottleneck_dim, out_dim, bias=False)) + self.last_layer.weight_g.data.fill_(1) + if norm_last_layer: + self.last_layer.weight_g.requires_grad = False + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + torch.nn.init.trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + + def forward(self, x): + vicr_out = self.mlp(x) + x = self.add_layer(vicr_out) + x = nn.functional.normalize(x, dim=-1, p=2) + x = self.last_layer(x) + return vicr_out, x + + +class Combine(nn.Module): + + def __init__(self, backbone, head): + super(Combine, self).__init__() + self.backbone = backbone + self.head = head + + def forward(self, x): + x = self.backbone(x) + output = self.head(x) + return output + + +@MODELS.register_module( + Tasks.speaker_verification, module_name=Models.rdino_tdnn_sv) +class SpeakerVerification_RDINO(TorchModel): + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + self.other_config = kwargs + if self.model_config['channel'] != 1024: + raise ValueError( + 'modelscope error: Currently only 1024-channel ecapa tdnn is supported.' + ) + + self.feature_dim = 80 + channels_config = [1024, 1024, 1024, 1024, 3072] + + self.embedding_model = ECAPA_TDNN( + self.feature_dim, channels=channels_config) + self.embedding_model = Combine(self.embedding_model, + RDINOHead(512, 65536, True)) + + pretrained_model_name = kwargs['pretrained_model'] + self.__load_check_point(pretrained_model_name) + + self.embedding_model.eval() + + def forward(self, audio): + assert len(audio.shape) == 2 and audio.shape[ + 0] == 1, 'modelscope error: the shape of input audio to model needs to be [1, T]' + # audio shape: [1, T] + feature = self.__extract_feature(audio) + embedding = self.embedding_model.backbone(feature) + + return embedding + + def __extract_feature(self, audio): + feature = Kaldi.fbank(audio, num_mel_bins=self.feature_dim) + feature = feature - feature.mean(dim=0, keepdim=True) + feature = feature.unsqueeze(0) + return feature + + def __load_check_point(self, pretrained_model_name, device=None): + if not device: + device = torch.device('cpu') + state_dict = torch.load( + os.path.join(self.model_dir, pretrained_model_name), + map_location=device) + state_dict_tea = { + k.replace('module.', ''): v + for k, v in state_dict['teacher'].items() + } + self.embedding_model.load_state_dict(state_dict_tea, strict=True) diff --git a/modelscope/pipelines/audio/speaker_verification_rdino_pipeline.py b/modelscope/pipelines/audio/speaker_verification_rdino_pipeline.py new file mode 100644 index 00000000..dd08ccf4 --- /dev/null +++ b/modelscope/pipelines/audio/speaker_verification_rdino_pipeline.py @@ -0,0 +1,110 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import io +from typing import Any, Dict, List, Union + +import soundfile as sf +import torch + +from modelscope.fileio import File +from modelscope.metainfo import Pipelines +from modelscope.outputs import OutputKeys +from modelscope.pipelines.base import InputModel, Pipeline +from modelscope.pipelines.builder import PIPELINES +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@PIPELINES.register_module( + Tasks.speaker_verification, + module_name=Pipelines.speaker_verification_rdino) +class RDINO_Pipeline(Pipeline): + """Speaker Verification Inference Pipeline + use `model` to create a Speaker Verification pipeline. + + Args: + model (SpeakerVerificationPipeline): A model instance, or a model local dir, or a model id in the model hub. + kwargs (dict, `optional`): + Extra kwargs passed into the pipeline's constructor. + Example: + >>> from modelscope.pipelines import pipeline + >>> from modelscope.utils.constant import Tasks + >>> p = pipeline( + >>> task=Tasks.speaker_verification, model='damo/speech_ecapa-tdnn_sv_en_voxceleb_16k') + >>> print(p([audio_1, audio_2])) + + """ + + def __init__(self, model: InputModel, **kwargs): + """use `model` to create a speaker verification pipeline for prediction + Args: + model (str): a valid offical model id + """ + super().__init__(model=model, **kwargs) + self.model_config = self.model.model_config + self.config = self.model.other_config + self.thr = self.config['yesOrno_thr'] + + def __call__(self, + in_audios: List[str], + thr: float = None) -> Dict[str, Any]: + if thr is not None: + self.thr = thr + if self.thr < -1 or self.thr > 1: + raise ValueError( + 'modelscope error: the thr value should be in [-1, 1], but found to be %f.' + % self.thr) + outputs = self.preprocess(in_audios) + outputs = self.forward(outputs) + outputs = self.postprocess(outputs) + + return outputs + + def forward(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + emb1 = self.model(inputs['data1']) + emb2 = self.model(inputs['data2']) + + return {'emb1': emb1, 'emb2': emb2} + + def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + score = self.compute_cos_similarity(inputs['emb1'], inputs['emb2']) + score = round(score, 5) + if score >= self.thr: + ans = 'yes' + else: + ans = 'no' + + return {OutputKeys.SCORE: score, OutputKeys.TEXT: ans} + + def preprocess(self, inputs: List[str], + **preprocess_params) -> Dict[str, Any]: + if len(inputs) != 2: + raise ValueError( + 'modelscope error: Two input audio files are required.') + output = {} + for i in range(len(inputs)): + if isinstance(inputs[i], str): + file_bytes = File.read(inputs[i]) + data, fs = sf.read(io.BytesIO(file_bytes), dtype='float32') + if len(data.shape) == 2: + data = data[:, 0] + if fs != self.model_config['sample_rate']: + raise ValueError( + 'modelscope error: Only support %d sample rate files' + % self.model_cfg['sample_rate']) + output['data%d' % + (i + 1)] = torch.from_numpy(data).unsqueeze(0) + else: + raise ValueError( + 'modelscope error: The input type is temporarily restricted to audio file address' + % i) + return output + + def compute_cos_similarity(self, emb1: torch.Tensor, + emb2: torch.Tensor) -> float: + assert len(emb1.shape) == 2 and len(emb2.shape) == 2 + cos = torch.nn.CosineSimilarity(dim=1, eps=1e-6) + cosine = cos(emb1, emb2) + return cosine.item() diff --git a/tests/pipelines/test_speaker_verification.py b/tests/pipelines/test_speaker_verification.py index 83d8aff3..236b952b 100644 --- a/tests/pipelines/test_speaker_verification.py +++ b/tests/pipelines/test_speaker_verification.py @@ -21,12 +21,17 @@ SPEAKER2_A_EN_16K_WAV = 'data/test/audios/speaker2_a_en_16k.wav' class SpeakerVerificationTest(unittest.TestCase, DemoCompatibilityCheck): ecapatdnn_voxceleb_16k_model_id = 'damo/speech_ecapa-tdnn_sv_en_voxceleb_16k' campplus_voxceleb_16k_model_id = 'damo/speech_campplus_sv_en_voxceleb_16k' + rdino_voxceleb_16k_model_id = 'damo/speech_rdino_ecapa_tdnn_sv_en_voxceleb_16k' def setUp(self) -> None: self.task = Tasks.speaker_verification - def run_pipeline(self, model_id: str, audios: List[str]) -> Dict[str, Any]: - p = pipeline(task=self.task, model=model_id) + def run_pipeline(self, + model_id: str, + audios: List[str], + model_revision=None) -> Dict[str, Any]: + p = pipeline( + task=self.task, model=model_id, model_revision=model_revision) result = p(audios) return result @@ -51,6 +56,16 @@ class SpeakerVerificationTest(unittest.TestCase, DemoCompatibilityCheck): print(result) self.assertTrue(OutputKeys.SCORE in result) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_speaker_verification_rdino_voxceleb_16k(self): + logger.info('Run speaker verification for rdino_voxceleb_16k model') + result = self.run_pipeline( + model_id=self.rdino_voxceleb_16k_model_id, + audios=[SPEAKER1_A_EN_16K_WAV, SPEAKER1_B_EN_16K_WAV], + model_revision='v1.0.1') + print(result) + self.assertTrue(OutputKeys.SCORE in result) + @unittest.skip('demo compatibility test is only enabled on a needed-basis') def test_demo_compatibility(self): self.compatibility_check() From 6d68f0ea64fbbc27d3043d47db9bfbc71bc0028d Mon Sep 17 00:00:00 2001 From: "bin.xue" Date: Fri, 28 Apr 2023 10:33:32 +0800 Subject: [PATCH 12/76] [to #42322933] add ONNX exporter for ans dfsmn --- data/test | 2 +- modelscope/exporters/audio/__init__.py | 22 +++++ .../exporters/audio/ans_dfsmn_exporter.py | 62 ++++++++++++++ .../test_export_speech_signal_process.py | 83 +++++++++++++++++++ 4 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 modelscope/exporters/audio/__init__.py create mode 100644 modelscope/exporters/audio/ans_dfsmn_exporter.py create mode 100644 tests/export/test_export_speech_signal_process.py diff --git a/data/test b/data/test index c117008c..0a61e00d 160000 --- a/data/test +++ b/data/test @@ -1 +1 @@ -Subproject commit c117008caa9dc447c208e9ed6bc11310512d4a3a +Subproject commit 0a61e00de4a4b529099b357cbb0b2af83ac2f31e diff --git a/modelscope/exporters/audio/__init__.py b/modelscope/exporters/audio/__init__.py new file mode 100644 index 00000000..883151cd --- /dev/null +++ b/modelscope/exporters/audio/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .ans_dfsmn_exporter import ANSDFSMNExporter +else: + _import_structure = { + 'ans_dfsmn_exporter': ['ANSDFSMNExporter'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/modelscope/exporters/audio/ans_dfsmn_exporter.py b/modelscope/exporters/audio/ans_dfsmn_exporter.py new file mode 100644 index 00000000..976f983f --- /dev/null +++ b/modelscope/exporters/audio/ans_dfsmn_exporter.py @@ -0,0 +1,62 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os + +import torch + +from modelscope.exporters.builder import EXPORTERS +from modelscope.exporters.torch_model_exporter import TorchModelExporter +from modelscope.metainfo import Models +from modelscope.utils.constant import ModelFile, Tasks + +INPUT_NAME = 'input' +OUTPUT_NAME = 'output' + + +@EXPORTERS.register_module( + Tasks.acoustic_noise_suppression, module_name=Models.speech_dfsmn_ans) +class ANSDFSMNExporter(TorchModelExporter): + + def export_onnx(self, output_dir: str, opset=9, **kwargs): + """Export the model as onnx format files. + + Args: + output_dir: The output dir. + opset: The version of the ONNX operator set to use. + kwargs: + device: The device used to forward. + Returns: + A dict containing the model key - model file path pairs. + """ + model = self.model if 'model' not in kwargs else kwargs.pop('model') + device_name = 'cpu' if 'device' not in kwargs else kwargs.pop('device') + model_bin_file = os.path.join(model.model_dir, + ModelFile.TORCH_MODEL_BIN_FILE) + if os.path.exists(model_bin_file): + checkpoint = torch.load(model_bin_file, map_location='cpu') + model.load_state_dict(checkpoint) + onnx_file = os.path.join(output_dir, ModelFile.ONNX_MODEL_FILE) + + with torch.no_grad(): + model.eval() + device = torch.device(device_name) + model.to(device) + model_script = torch.jit.script(model) + fbank_input = torch.zeros((1, 3, 120), dtype=torch.float32) + torch.onnx.export( + model_script, + fbank_input, + onnx_file, + opset_version=opset, + input_names=[INPUT_NAME], + output_names=[OUTPUT_NAME], + dynamic_axes={ + INPUT_NAME: { + 0: 'batch_size', + 1: 'number_of_frame' + }, + OUTPUT_NAME: { + 0: 'batch_size', + 1: 'number_of_frame' + } + }) + return {'model': onnx_file} diff --git a/tests/export/test_export_speech_signal_process.py b/tests/export/test_export_speech_signal_process.py new file mode 100644 index 00000000..d3f6fe14 --- /dev/null +++ b/tests/export/test_export_speech_signal_process.py @@ -0,0 +1,83 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +import pickle +import shutil +import tempfile +import unittest + +import torch + +from modelscope.exporters import Exporter +from modelscope.models import Model +from modelscope.utils.logger import get_logger +from modelscope.utils.regress_test_utils import (compare_arguments_nested, + numpify_tensor_nested) +from modelscope.utils.test_utils import test_level + +INPUT_PKL = 'data/test/audios/input.pkl' + +INPUT_NAME = 'input' +OUTPUT_NAME = 'output' + +logger = get_logger() + + +class ExportSpeechSignalProcessTest(unittest.TestCase): + + def setUp(self): + print(('Testing %s.%s' % (type(self).__name__, self._testMethodName))) + self.tmp_dir = tempfile.TemporaryDirectory().name + if not os.path.exists(self.tmp_dir): + os.makedirs(self.tmp_dir) + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + super().tearDown() + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_export_ans_dfsmn(self): + model_id = 'damo/speech_dfsmn_ans_psm_48k_causal' + model = Model.from_pretrained(model_id) + onnx_info = Exporter.from_model(model).export_onnx( + output_dir=self.tmp_dir) + + with open(os.path.join(os.getcwd(), INPUT_PKL), 'rb') as f: + fbank_input = pickle.load(f).cpu() + self.assertTrue( + self._validate_onnx_model(fbank_input, model, onnx_info['model']), + 'export onnx failed because of validation error.') + + @staticmethod + def _validate_onnx_model(dummy_inputs, model, output): + try: + import onnx + import onnxruntime as ort + except ImportError: + logger.warning( + 'Cannot validate the exported onnx file, because ' + 'the installation of onnx or onnxruntime cannot be found') + return + onnx_model = onnx.load(output) + onnx.checker.check_model(onnx_model) + ort_session = ort.InferenceSession(output) + with torch.no_grad(): + model.eval() + outputs_origin = model.forward(dummy_inputs) + outputs_origin = numpify_tensor_nested(outputs_origin) + + input_feed = {INPUT_NAME: dummy_inputs.numpy()} + outputs = ort_session.run( + None, + input_feed, + ) + outputs = numpify_tensor_nested(outputs[0]) + + print(outputs) + print(outputs_origin) + return compare_arguments_nested('Onnx model output match failed', + outputs, outputs_origin) + + +if __name__ == '__main__': + unittest.main() From d772d96cf5afa72930b4f8920e8db26a45d8421c Mon Sep 17 00:00:00 2001 From: "mulin.lyh" Date: Sat, 6 May 2023 18:01:02 +0800 Subject: [PATCH 13/76] [to #49147498]feat: support python3.8 --- modelscope/models/audio/ans/conv_stft.py | 4 +- .../models/cv/body_2d_keypoints/hrnet_v2.py | 2 +- .../cv/cartoon/facelib/face_landmark.py | 2 +- .../cv/crowd_counting/hrnet_aspp_relu.py | 2 +- .../peppa_pig_face/face_landmark.py | 2 +- .../models/human_segmenter.py | 2 +- .../models/cv/human_reconstruction/utils.py | 4 +- .../utils/voc_register.py | 4 +- .../postprocess_utils.py | 12 ++--- .../colmap2mvsnet.py | 2 +- .../depth_filter.py | 2 +- .../semantic_seg_model.py | 2 +- .../ptsemseg/hrnet_backnone.py | 2 +- .../ptsemseg/hrnet_super_and_ocr.py | 4 +- .../cv/nerf_recon_acc/network/segmenter.py | 2 +- .../cv/object_detection_3d/depe/result_vis.py | 6 +-- .../cv/open_vocabulary_detection_vild/vild.py | 3 +- .../networks/layers.py | 2 +- .../cv/video_depth_estimation/utils/depth.py | 2 +- .../utils/scene_change_detection.py | 2 +- .../tracker/matching.py | 8 ++-- .../tracker/multitracker.py | 2 +- .../mmr/models/clip_for_mm_video_embedding.py | 4 +- modelscope/models/nlp/mglm/blocklm_utils.py | 45 +++++++++---------- .../models/nlp/mglm/data_utils/datasets.py | 4 +- modelscope/models/nlp/mglm/test/test_block.py | 2 +- .../models/nlp/mglm/test/test_rel_shift.py | 2 +- .../science/unifold/data/msa_pairing.py | 2 +- .../models/science/unifold/msa/templates.py | 4 +- .../cv/face_reconstruction_pipeline.py | 2 +- .../pipelines/cv/image_matting_pipeline.py | 2 +- .../cv/image_style_transfer_pipeline.py | 4 +- .../pipelines/cv/skin_retouching_pipeline.py | 2 +- .../pipelines/cv/tbs_detection_pipeline.py | 2 +- modelscope/utils/regress_test_utils.py | 4 +- modelscope/utils/test_utils.py | 8 ++-- requirements/framework.txt | 2 + 37 files changed, 79 insertions(+), 83 deletions(-) diff --git a/modelscope/models/audio/ans/conv_stft.py b/modelscope/models/audio/ans/conv_stft.py index 4b393a4c..3d37f1aa 100644 --- a/modelscope/models/audio/ans/conv_stft.py +++ b/modelscope/models/audio/ans/conv_stft.py @@ -39,7 +39,7 @@ class ConvSTFT(nn.Module): super(ConvSTFT, self).__init__() if fft_len is None: - self.fft_len = np.int(2**np.ceil(np.log2(win_len))) + self.fft_len = int(2**np.ceil(np.log2(win_len))) else: self.fft_len = fft_len @@ -78,7 +78,7 @@ class ConviSTFT(nn.Module): fix=True): super(ConviSTFT, self).__init__() if fft_len is None: - self.fft_len = np.int(2**np.ceil(np.log2(win_len))) + self.fft_len = int(2**np.ceil(np.log2(win_len))) else: self.fft_len = fft_len kernel, window = init_kernels( diff --git a/modelscope/models/cv/body_2d_keypoints/hrnet_v2.py b/modelscope/models/cv/body_2d_keypoints/hrnet_v2.py index ebd69adb..19e426b2 100644 --- a/modelscope/models/cv/body_2d_keypoints/hrnet_v2.py +++ b/modelscope/models/cv/body_2d_keypoints/hrnet_v2.py @@ -72,7 +72,7 @@ class PoseHighResolutionNetV2(TorchModel): self.stage4, pre_stage_channels = self._make_stage( self.stage4_cfg, num_channels, multi_scale_output=True) """final four layers""" - last_inp_channels = np.int(np.sum(pre_stage_channels)) + last_inp_channels = int(np.sum(pre_stage_channels)) self.final_layer = nn.Sequential( nn.Conv2d( in_channels=last_inp_channels, diff --git a/modelscope/models/cv/cartoon/facelib/face_landmark.py b/modelscope/models/cv/cartoon/facelib/face_landmark.py index 3b7cc1b9..3c53f3a6 100644 --- a/modelscope/models/cv/cartoon/facelib/face_landmark.py +++ b/modelscope/models/cv/cartoon/facelib/face_landmark.py @@ -81,7 +81,7 @@ class FaceLandmark: bbox[2] = center[0] + one_edge // 2 bbox[3] = center[1] + one_edge // 2 - bbox = bbox.astype(np.int) + bbox = bbox.astype(int) crop_image = bimg[bbox[1]:bbox[3], bbox[0]:bbox[2], :] h, w, _ = crop_image.shape crop_image = cv2.resize( diff --git a/modelscope/models/cv/crowd_counting/hrnet_aspp_relu.py b/modelscope/models/cv/crowd_counting/hrnet_aspp_relu.py index 0d1bd3ca..64f40da0 100644 --- a/modelscope/models/cv/crowd_counting/hrnet_aspp_relu.py +++ b/modelscope/models/cv/crowd_counting/hrnet_aspp_relu.py @@ -356,7 +356,7 @@ class HighResolutionNet(nn.Module): num_channels) self.stage3, pre_stage_channels = self._make_stage( self.stage3_cfg, num_channels) - last_inp_channels = np.int(np.sum(pre_stage_channels)) + 256 + last_inp_channels = int(np.sum(pre_stage_channels)) + 256 self.redc_layer = nn.Sequential( nn.Conv2d( in_channels=last_inp_channels, diff --git a/modelscope/models/cv/face_detection/peppa_pig_face/face_landmark.py b/modelscope/models/cv/face_detection/peppa_pig_face/face_landmark.py index 03a3b5b7..e7e2ddaf 100644 --- a/modelscope/models/cv/face_detection/peppa_pig_face/face_landmark.py +++ b/modelscope/models/cv/face_detection/peppa_pig_face/face_landmark.py @@ -82,7 +82,7 @@ class FaceLandmark: bbox[2] = center[0] + one_edge // 2 bbox[3] = center[1] + one_edge // 2 - bbox = bbox.astype(np.int) + bbox = bbox.astype(int) crop_image = bimg[bbox[1]:bbox[3], bbox[0]:bbox[2], :] h, w, _ = crop_image.shape crop_image = cv2.resize(crop_image, diff --git a/modelscope/models/cv/human_reconstruction/models/human_segmenter.py b/modelscope/models/cv/human_reconstruction/models/human_segmenter.py index 3f0261e7..29bf6f70 100644 --- a/modelscope/models/cv/human_reconstruction/models/human_segmenter.py +++ b/modelscope/models/cv/human_reconstruction/models/human_segmenter.py @@ -31,7 +31,7 @@ class human_segmenter(object): img = np.dstack((img, img, img)) elif img.shape[2] == 4: img = img[:, :, :3] - img = img.astype(np.float) + img = img.astype(float) return img def run(self, img): diff --git a/modelscope/models/cv/human_reconstruction/utils.py b/modelscope/models/cv/human_reconstruction/utils.py index 45653dc6..67e1efdb 100644 --- a/modelscope/models/cv/human_reconstruction/utils.py +++ b/modelscope/models/cv/human_reconstruction/utils.py @@ -69,8 +69,8 @@ def eval_grid(coords, num_samples=512 * 512 * 512): resolution = coords.shape[1:4] sdf = np.zeros(resolution) - dirty = np.ones(resolution, dtype=np.bool) - grid_mask = np.zeros(resolution, dtype=np.bool) + dirty = np.ones(resolution, dtype=bool) + grid_mask = np.zeros(resolution, dtype=bool) reso = resolution[0] // init_resolution while reso > 0: diff --git a/modelscope/models/cv/image_defrcn_fewshot/utils/voc_register.py b/modelscope/models/cv/image_defrcn_fewshot/utils/voc_register.py index 7a94066e..0b043493 100644 --- a/modelscope/models/cv/image_defrcn_fewshot/utils/voc_register.py +++ b/modelscope/models/cv/image_defrcn_fewshot/utils/voc_register.py @@ -163,7 +163,7 @@ def load_filtered_voc_instances(name: str, root: str, dirname: str, split: str, os.path.join(split_dir, 'box_{}shot_{}_train.txt'.format(shot, cls))) as f: - fileids_ = np.loadtxt(f, dtype=np.str).tolist() + fileids_ = np.loadtxt(f, dtype=np.str_).tolist() if isinstance(fileids_, str): fileids_ = [fileids_] fileids_ = [ @@ -219,7 +219,7 @@ def load_filtered_voc_instances(name: str, root: str, dirname: str, split: str, with PathManager.open( os.path.join(root, dirname, 'ImageSets', 'Main', split + '.txt')) as f: - fileids = np.loadtxt(f, dtype=np.str) + fileids = np.loadtxt(f, dtype=np.str_) for fileid in fileids: anno_file = os.path.join(root, dirname, 'Annotations', diff --git a/modelscope/models/cv/image_instance_segmentation/postprocess_utils.py b/modelscope/models/cv/image_instance_segmentation/postprocess_utils.py index fdbb2fb0..aad7d8e9 100644 --- a/modelscope/models/cv/image_instance_segmentation/postprocess_utils.py +++ b/modelscope/models/cv/image_instance_segmentation/postprocess_utils.py @@ -108,16 +108,16 @@ def get_img_ins_seg_result(img_seg_result=None, for seg_result in img_seg_result: box = [ - np.int(seg_result[0]), - np.int(seg_result[1]), - np.int(seg_result[2]), - np.int(seg_result[3]) + int(seg_result[0]), + int(seg_result[1]), + int(seg_result[2]), + int(seg_result[3]) ] - score = np.float(seg_result[4]) + score = float(seg_result[4]) category = seg_result[5] mask = np.array(seg_result[6], order='F', dtype='uint8') - mask = mask.astype(np.float) + mask = mask.astype(float) results_dict[OutputKeys.BOXES].append(box) results_dict[OutputKeys.MASKS].append(mask) diff --git a/modelscope/models/cv/image_mvs_depth_estimation/colmap2mvsnet.py b/modelscope/models/cv/image_mvs_depth_estimation/colmap2mvsnet.py index feda4430..37d92c13 100644 --- a/modelscope/models/cv/image_mvs_depth_estimation/colmap2mvsnet.py +++ b/modelscope/models/cv/image_mvs_depth_estimation/colmap2mvsnet.py @@ -382,7 +382,7 @@ def processing_single_scene(args): points3d[p3d_id].xyz[0], points3d[p3d_id].xyz[1], points3d[p3d_id].xyz[2], 1 ]) - zs.append(np.asscalar(transformed[2])) + zs.append(transformed[2].item()) zs_sorted = sorted(zs) # relaxed depth range max_ratio = 0.1 diff --git a/modelscope/models/cv/image_mvs_depth_estimation/depth_filter.py b/modelscope/models/cv/image_mvs_depth_estimation/depth_filter.py index 16cdedf4..4ef6275a 100644 --- a/modelscope/models/cv/image_mvs_depth_estimation/depth_filter.py +++ b/modelscope/models/cv/image_mvs_depth_estimation/depth_filter.py @@ -40,7 +40,7 @@ def read_mask(filename): # save a binary mask def save_mask(filename, mask): - assert mask.dtype == np.bool + assert mask.dtype == bool mask = mask.astype(np.uint8) * 255 Image.fromarray(mask).save(filename) diff --git a/modelscope/models/cv/image_semantic_segmentation/semantic_seg_model.py b/modelscope/models/cv/image_semantic_segmentation/semantic_seg_model.py index 2b38ebad..455f29fb 100644 --- a/modelscope/models/cv/image_semantic_segmentation/semantic_seg_model.py +++ b/modelscope/models/cv/image_semantic_segmentation/semantic_seg_model.py @@ -60,7 +60,7 @@ class SemanticSegmentation(TorchModel): ids = ids[legal_indices] segms = (semantic_result[None] == ids[:, None, None]) - masks = [it.astype(np.int) for it in segms] + masks = [it.astype(int) for it in segms] labels_txt = np.array(self.CLASSES)[ids].tolist() results = { diff --git a/modelscope/models/cv/image_skychange/ptsemseg/hrnet_backnone.py b/modelscope/models/cv/image_skychange/ptsemseg/hrnet_backnone.py index 66429d67..8fcb6625 100644 --- a/modelscope/models/cv/image_skychange/ptsemseg/hrnet_backnone.py +++ b/modelscope/models/cv/image_skychange/ptsemseg/hrnet_backnone.py @@ -458,7 +458,7 @@ class HrnetBackBone(nn.Module): self.stage4, pre_stage_channels = self._make_stage( self.stage4_cfg, num_channels, multi_scale_output=True) - self.backbone_last_inp_channels = np.int(np.sum(pre_stage_channels)) + self.backbone_last_inp_channels = int(np.sum(pre_stage_channels)) def _make_transition_layer(self, num_channels_pre_layer, num_channels_cur_layer): diff --git a/modelscope/models/cv/image_skychange/ptsemseg/hrnet_super_and_ocr.py b/modelscope/models/cv/image_skychange/ptsemseg/hrnet_super_and_ocr.py index 09768451..5dbef66e 100644 --- a/modelscope/models/cv/image_skychange/ptsemseg/hrnet_super_and_ocr.py +++ b/modelscope/models/cv/image_skychange/ptsemseg/hrnet_super_and_ocr.py @@ -259,7 +259,7 @@ class HrnetSuperAndOcr(HrnetBackBone): num_channels = [64, last_inp_channels] self.stage_super, super_stage_channels = self._make_stage( self.super_dict, num_channels) - last_inp_channels = np.int(np.sum(super_stage_channels)) + last_inp_channels = int(np.sum(super_stage_channels)) if self.is_contain_aspp: aspp_param = kwargs['aspp'] @@ -372,7 +372,7 @@ class HrnetSuperAndOcr(HrnetBackBone): num_channels = [64, ocr_mid_channels] self.stage_super, super_stage_channels = self._make_stage( self.super_dict, num_channels) - last_inp_channels = np.int(np.sum(super_stage_channels)) + last_inp_channels = int(np.sum(super_stage_channels)) self.cls_head = nn.Sequential( nn.Conv2d( diff --git a/modelscope/models/cv/nerf_recon_acc/network/segmenter.py b/modelscope/models/cv/nerf_recon_acc/network/segmenter.py index d71b9f16..e3d0ca8d 100644 --- a/modelscope/models/cv/nerf_recon_acc/network/segmenter.py +++ b/modelscope/models/cv/nerf_recon_acc/network/segmenter.py @@ -31,7 +31,7 @@ class ObjectSegmenter(object): elif img.shape[2] == 4: img = img[:, :, :3] img = img[:, :, ::-1] - img = img.astype(np.float) + img = img.astype(float) return img def run_mask(self, img): diff --git a/modelscope/models/cv/object_detection_3d/depe/result_vis.py b/modelscope/models/cv/object_detection_3d/depe/result_vis.py index d577ab68..d567654e 100644 --- a/modelscope/models/cv/object_detection_3d/depe/result_vis.py +++ b/modelscope/models/cv/object_detection_3d/depe/result_vis.py @@ -30,7 +30,7 @@ def depth2color(depth): if gray == 1: return tuple(colors[-1].tolist()) num_rank = len(colors) - 1 - rank = np.floor(gray * num_rank).astype(np.int) + rank = np.floor(gray * num_rank).astype(int) diff = (gray - rank / num_rank) * num_rank tmp = colors[rank + 1] - colors[rank] return tuple((colors[rank] + tmp * diff).tolist()) @@ -136,7 +136,7 @@ def plot_result(res_path, l2g = get_lidar2global(infos) corners_lidar = corners_global @ np.linalg.inv(l2g).T corners_lidar = corners_lidar[:, :3] - pred_flag = np.ones((corners_lidar.shape[0] // 8, ), dtype=np.bool) + pred_flag = np.ones((corners_lidar.shape[0] // 8, ), dtype=bool) scores = [ pred_res[rid]['detection_score'] for rid in range(len(pred_res)) ] @@ -151,7 +151,7 @@ def plot_result(res_path, origin=(0.5, 0.5, 0.5)).corners.numpy().reshape(-1, 3) corners_lidar = np.concatenate([corners_lidar, corners_lidar_gt], axis=0) - gt_flag = np.ones((corners_lidar_gt.shape[0] // 8), dtype=np.bool) + gt_flag = np.ones((corners_lidar_gt.shape[0] // 8), dtype=bool) pred_flag = np.concatenate( [pred_flag, np.logical_not(gt_flag)], axis=0) scores = scores + [0 for _ in range(infos['gt_boxes'].shape[0])] diff --git a/modelscope/models/cv/open_vocabulary_detection_vild/vild.py b/modelscope/models/cv/open_vocabulary_detection_vild/vild.py index 999ec27a..2aea0593 100644 --- a/modelscope/models/cv/open_vocabulary_detection_vild/vild.py +++ b/modelscope/models/cv/open_vocabulary_detection_vild/vild.py @@ -176,8 +176,7 @@ class OpenVocabularyDetectionViLD(Model): # Filter out invalid rois (nmsed rois) valid_indices = np.where( np.logical_and( - np.isin( - np.arange(len(roi_scores), dtype=np.int), nmsed_indices), + np.isin(np.arange(len(roi_scores), dtype=int), nmsed_indices), np.logical_and( np.logical_not(np.all(roi_boxes == 0., axis=-1)), np.logical_and(roi_scores >= min_rpn_score_thresh, diff --git a/modelscope/models/cv/panorama_depth_estimation/networks/layers.py b/modelscope/models/cv/panorama_depth_estimation/networks/layers.py index 99e166aa..52fb3d39 100644 --- a/modelscope/models/cv/panorama_depth_estimation/networks/layers.py +++ b/modelscope/models/cv/panorama_depth_estimation/networks/layers.py @@ -72,7 +72,7 @@ class Cube2Equirec(nn.Module): self.equ_h, 0), 3 * self.equ_w // 8, 1) # Prepare ceil mask - mask = np.zeros((self.equ_h, self.equ_w // 4), np.bool) + mask = np.zeros((self.equ_h, self.equ_w // 4), bool) idx = np.linspace(-np.pi, np.pi, self.equ_w // 4) / 4 idx = self.equ_h // 2 - np.round( np.arctan(np.cos(idx)) * self.equ_h / np.pi).astype(int) diff --git a/modelscope/models/cv/video_depth_estimation/utils/depth.py b/modelscope/models/cv/video_depth_estimation/utils/depth.py index e9f287e7..5fbf6aa6 100644 --- a/modelscope/models/cv/video_depth_estimation/utils/depth.py +++ b/modelscope/models/cv/video_depth_estimation/utils/depth.py @@ -29,7 +29,7 @@ def load_depth(file): elif file.endswith('png'): depth_png = np.array(load_image(file), dtype=int) assert (np.max(depth_png) > 255), 'Wrong .png depth file' - return depth_png.astype(np.float) / 256. + return depth_png.astype(float) / 256. else: raise NotImplementedError('Depth extension not supported.') diff --git a/modelscope/models/cv/video_frame_interpolation/utils/scene_change_detection.py b/modelscope/models/cv/video_frame_interpolation/utils/scene_change_detection.py index 4cbe60a7..379fe855 100644 --- a/modelscope/models/cv/video_frame_interpolation/utils/scene_change_detection.py +++ b/modelscope/models/cv/video_frame_interpolation/utils/scene_change_detection.py @@ -85,7 +85,7 @@ def do_scene_detect(F01_tensor, F10_tensor, img0_tensor, img1_tensor): img_diff = ori_img.float() - ref_img.float() img_diff = torch.abs(img_diff) - kernel = np.ones([8, 8], np.float) / 64 + kernel = np.ones([8, 8], float) / 64 kernel = torch.FloatTensor(kernel).to(device).unsqueeze(0).unsqueeze(0) diff = F.conv2d(img_diff, kernel, padding=4) diff --git a/modelscope/models/cv/video_multi_object_tracking/tracker/matching.py b/modelscope/models/cv/video_multi_object_tracking/tracker/matching.py index 45d2f5c0..e5c2e8a9 100644 --- a/modelscope/models/cv/video_multi_object_tracking/tracker/matching.py +++ b/modelscope/models/cv/video_multi_object_tracking/tracker/matching.py @@ -27,7 +27,7 @@ def linear_assignment(cost_matrix, thresh): def ious(atlbrs, btlbrs): - ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float) + ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=float) if ious.size == 0: return ious @@ -60,13 +60,13 @@ def embedding_distance(tracks, detections, metric='cosine'): cost_matrix: np.ndarray """ - cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float) + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=float) if cost_matrix.size == 0: return cost_matrix det_features = np.asarray([track.curr_feat for track in detections], - dtype=np.float) + dtype=float) track_features = np.asarray([track.smooth_feat for track in tracks], - dtype=np.float) + dtype=float) cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) return cost_matrix diff --git a/modelscope/models/cv/video_multi_object_tracking/tracker/multitracker.py b/modelscope/models/cv/video_multi_object_tracking/tracker/multitracker.py index 1dc3297f..d38477b7 100644 --- a/modelscope/models/cv/video_multi_object_tracking/tracker/multitracker.py +++ b/modelscope/models/cv/video_multi_object_tracking/tracker/multitracker.py @@ -28,7 +28,7 @@ class STrack(BaseTrack): def __init__(self, tlwh, score, temp_feat, buffer_size=30): # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) + self._tlwh = np.asarray(tlwh, dtype=float) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False diff --git a/modelscope/models/multi_modal/mmr/models/clip_for_mm_video_embedding.py b/modelscope/models/multi_modal/mmr/models/clip_for_mm_video_embedding.py index 813f750e..743c049a 100644 --- a/modelscope/models/multi_modal/mmr/models/clip_for_mm_video_embedding.py +++ b/modelscope/models/multi_modal/mmr/models/clip_for_mm_video_embedding.py @@ -128,13 +128,13 @@ class VideoCLIPForMultiModalEmbedding(TorchModel): local_transform, s=None, e=None): - video_mask = np.zeros(self.max_frames, dtype=np.long) + video_mask = np.zeros(self.max_frames, dtype=int) max_video_length = 0 # T x 3 x H x W video = np.zeros((self.max_frames, 3, rawVideoExtractor.size, rawVideoExtractor.size), - dtype=np.float) + dtype=float) if s is None: start_time, end_time = None, None diff --git a/modelscope/models/nlp/mglm/blocklm_utils.py b/modelscope/models/nlp/mglm/blocklm_utils.py index b05cd2c2..e75aea92 100644 --- a/modelscope/models/nlp/mglm/blocklm_utils.py +++ b/modelscope/models/nlp/mglm/blocklm_utils.py @@ -212,10 +212,10 @@ class ConstructBlockStrategy: block_spans, rng, task='bert'): - position_ids = np.arange(len(tokens), dtype=np.long) + position_ids = np.arange(len(tokens), dtype=int) targets = copy.deepcopy(tokens) mask_id = self.tokenizer.get_command('MASK').Id - mlm_masks = np.zeros(len(tokens), dtype=np.long) + mlm_masks = np.zeros(len(tokens), dtype=int) for start, end in block_spans: for idx in range(start, end): tokens[idx] = mask_id @@ -231,7 +231,7 @@ class ConstructBlockStrategy: rng, task='bert'): text_length = len(tokens) - position_ids = np.ones(len(tokens), dtype=np.long) + position_ids = np.ones(len(tokens), dtype=int) for start, end in block_spans: position_ids[start + 1:end] = 0 position_ids = np.cumsum(position_ids) - 1 @@ -270,7 +270,7 @@ class ConstructBlockStrategy: (end - start + 1)) if self.block_position_encoding: target_block_position_ids.append( - np.arange(1, end - start + 2, dtype=np.long)) + np.arange(1, end - start + 2, dtype=int)) else: target_block_position_ids.append([1] * (end - start + 1)) block_spans.sort(key=lambda x: x[0]) @@ -307,7 +307,7 @@ class ConstructBlockStrategy: target_tokens = target_tokens + [ self.tokenizer.get_command('eop').Id ] - loss_masks = np.ones(len(target_tokens), dtype=np.long) + loss_masks = np.ones(len(target_tokens), dtype=int) return source_tokens, target_tokens, loss_masks else: tokens = np.concatenate(source_tokens + target_tokens) @@ -326,12 +326,12 @@ class ConstructBlockStrategy: for pos in mask_pos: tokens[pos] = self.tokenizer.get_command('dBLOCK').Id targets = np.concatenate(source_tokens + targets) - loss_masks = np.ones(len(tokens), dtype=np.long) + loss_masks = np.ones(len(tokens), dtype=int) loss_masks[:source_length] = 0 position_ids = np.concatenate(source_position_ids + target_position_ids) block_position_ids = np.concatenate( - [np.zeros(source_length, dtype=np.long)] + [np.zeros(source_length, dtype=int)] + target_block_position_ids) position_ids = np.stack([position_ids, block_position_ids], axis=0) if attention_mask is not None: @@ -539,22 +539,21 @@ class ConstructBlockStrategy: (source_tokens, [self.generation_mask], target_tokens)) loss_masks = np.concatenate( (np.zeros(len(source_tokens) + 1, - dtype=np.long), target_masks)) + dtype=int), target_masks)) token_batch.append(tokens) target_batch.append(targets) loss_mask_batch.append(loss_masks) position_ids = np.arange( - len(source_tokens) + len(target_tokens) + 1, - dtype=np.long) + len(source_tokens) + len(target_tokens) + 1, dtype=int) position_ids[len(source_tokens) + 1:] = len(source_tokens) if self.block_position_encoding: block_position_ids = np.concatenate( - (np.zeros(len(source_tokens), dtype=np.long), - np.arange(len(target_tokens) + 1, dtype=np.long))) + (np.zeros(len(source_tokens), dtype=int), + np.arange(len(target_tokens) + 1, dtype=int))) else: block_position_ids = np.concatenate( - (np.zeros(len(source_tokens) + 1, dtype=np.long), - np.ones(len(target_tokens) + 1, dtype=np.long))) + (np.zeros(len(source_tokens) + 1, dtype=int), + np.ones(len(target_tokens) + 1, dtype=int))) position_id_batch.append( np.stack([position_ids, block_position_ids], axis=0)) else: @@ -597,27 +596,25 @@ class ConstructBlockStrategy: max_length = max(seq_lengths) token_batch = [ np.concatenate( - (tokens, np.zeros(max_length - len(tokens), - dtype=np.long))) + (tokens, np.zeros(max_length - len(tokens), dtype=int))) for tokens in token_batch ] target_batch = [ np.concatenate( - (targets, - np.zeros(max_length - len(targets), dtype=np.long))) + (targets, np.zeros(max_length - len(targets), dtype=int))) for targets in target_batch ] loss_mask_batch = [ np.concatenate( (loss_masks, - np.zeros(max_length - len(loss_masks), dtype=np.long))) + np.zeros(max_length - len(loss_masks), dtype=int))) for loss_masks in loss_mask_batch ] position_id_batch = [ - np.concatenate((position_ids, - np.zeros( - (2, max_length - position_ids.shape[1]), - dtype=np.long)), - axis=1) for position_ids in position_id_batch + np.concatenate( + (position_ids, + np.zeros( + (2, max_length - position_ids.shape[1]), dtype=int)), + axis=1) for position_ids in position_id_batch ] return token_batch, target_batch, loss_mask_batch, position_id_batch diff --git a/modelscope/models/nlp/mglm/data_utils/datasets.py b/modelscope/models/nlp/mglm/data_utils/datasets.py index 39ffaea3..37bfbcc2 100644 --- a/modelscope/models/nlp/mglm/data_utils/datasets.py +++ b/modelscope/models/nlp/mglm/data_utils/datasets.py @@ -583,8 +583,8 @@ class XLDataset(data.Dataset): def getidx(self, idx): tokens, targets, loss_masks = [], [], [] attention_mask = np.concatenate( - (np.zeros((self.max_seq_len, self.mem_len), dtype=np.long), - np.ones((self.max_seq_len, self.max_seq_len), dtype=np.long)), + (np.zeros((self.max_seq_len, self.mem_len), dtype=int), + np.ones((self.max_seq_len, self.max_seq_len), dtype=int)), axis=1) sample_idx = bisect_right(self.indices, idx * self.max_seq_len) last_end = 0 if sample_idx == 0 else self.indices[sample_idx - 1] diff --git a/modelscope/models/nlp/mglm/test/test_block.py b/modelscope/models/nlp/mglm/test/test_block.py index ed4225da..eb630835 100644 --- a/modelscope/models/nlp/mglm/test/test_block.py +++ b/modelscope/models/nlp/mglm/test/test_block.py @@ -28,7 +28,7 @@ def main(): counts = np.array([0] * 10) for _ in range(10000): spans = strategy.sample_span_in_document( - np.array([1, 2, 3, 0, 4, 5, 6, 7, 9, 0], dtype=np.long), [1, 1], + np.array([1, 2, 3, 0, 4, 5, 6, 7, 9, 0], dtype=int), [1, 1], random.Random()) for start, end in spans: counts[start:end] += 1 diff --git a/modelscope/models/nlp/mglm/test/test_rel_shift.py b/modelscope/models/nlp/mglm/test/test_rel_shift.py index 00cbb9fe..ad68b15e 100644 --- a/modelscope/models/nlp/mglm/test/test_rel_shift.py +++ b/modelscope/models/nlp/mglm/test/test_rel_shift.py @@ -17,7 +17,7 @@ def main(): num_iters=300000, decay_style='cosine', decay_ratio=0.1) - steps = np.arange(0, 400000, 10, dtype=np.long) + steps = np.arange(0, 400000, 10, dtype=int) rates = [] for step in steps: lr_scheduler.num_iters = step diff --git a/modelscope/models/science/unifold/data/msa_pairing.py b/modelscope/models/science/unifold/data/msa_pairing.py index cc65962c..77c4e9a6 100644 --- a/modelscope/models/science/unifold/data/msa_pairing.py +++ b/modelscope/models/science/unifold/data/msa_pairing.py @@ -115,7 +115,7 @@ def pad_features(feature: np.ndarray, feature_name: str) -> np.ndarray: Returns: The feature with an additional padding row. """ - assert feature.dtype != np.dtype(np.string_) + assert feature.dtype != np.dtype(np.str_) if feature_name in ( 'msa_all_seq', 'msa_mask_all_seq', diff --git a/modelscope/models/science/unifold/msa/templates.py b/modelscope/models/science/unifold/msa/templates.py index d1ff8cf1..f2d3d79c 100644 --- a/modelscope/models/science/unifold/msa/templates.py +++ b/modelscope/models/science/unifold/msa/templates.py @@ -1100,9 +1100,9 @@ class HmmsearchHitFeaturizer(TemplateHitFeaturizer): np.zeros((1, num_res, residue_constants.atom_type_num, 3), np.float32), 'template_domain_names': - np.array([''.encode()], dtype=np.object), + np.array([''.encode()], dtype=np.object_), 'template_sequence': - np.array([''.encode()], dtype=np.object), + np.array([''.encode()], dtype=np.object_), 'template_sum_probs': np.array([0], dtype=np.float32), } diff --git a/modelscope/pipelines/cv/face_reconstruction_pipeline.py b/modelscope/pipelines/cv/face_reconstruction_pipeline.py index f8240fc0..b9a8e320 100644 --- a/modelscope/pipelines/cv/face_reconstruction_pipeline.py +++ b/modelscope/pipelines/cv/face_reconstruction_pipeline.py @@ -134,7 +134,7 @@ class FaceReconstructionPipeline(Pipeline): img = LoadImage.convert_to_ndarray(input) if len(img.shape) == 2: img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) - img = img.astype(np.float) + img = img.astype(float) result = {'img': img} return result diff --git a/modelscope/pipelines/cv/image_matting_pipeline.py b/modelscope/pipelines/cv/image_matting_pipeline.py index 5f5d1d56..bee655c5 100644 --- a/modelscope/pipelines/cv/image_matting_pipeline.py +++ b/modelscope/pipelines/cv/image_matting_pipeline.py @@ -53,7 +53,7 @@ class ImageMattingPipeline(Pipeline): def preprocess(self, input: Input) -> Dict[str, Any]: img = LoadImage.convert_to_ndarray(input) - img = img.astype(np.float) + img = img.astype(float) result = {'img': img} return result diff --git a/modelscope/pipelines/cv/image_style_transfer_pipeline.py b/modelscope/pipelines/cv/image_style_transfer_pipeline.py index e5fd0d48..49a0bff0 100644 --- a/modelscope/pipelines/cv/image_style_transfer_pipeline.py +++ b/modelscope/pipelines/cv/image_style_transfer_pipeline.py @@ -73,12 +73,12 @@ class ImageStyleTransferPipeline(Pipeline): content = LoadImage.convert_to_ndarray(content) if len(content.shape) == 2: content = cv2.cvtColor(content, cv2.COLOR_GRAY2BGR) - content_img = content.astype(np.float) + content_img = content.astype(float) style_img = LoadImage.convert_to_ndarray(style) if len(style_img.shape) == 2: style_img = cv2.cvtColor(style_img, cv2.COLOR_GRAY2BGR) - style_img = style_img.astype(np.float) + style_img = style_img.astype(float) result = {'content': content_img, 'style': style_img} return result diff --git a/modelscope/pipelines/cv/skin_retouching_pipeline.py b/modelscope/pipelines/cv/skin_retouching_pipeline.py index b2b5f4ca..da9b912f 100644 --- a/modelscope/pipelines/cv/skin_retouching_pipeline.py +++ b/modelscope/pipelines/cv/skin_retouching_pipeline.py @@ -105,7 +105,7 @@ class SkinRetouchingPipeline(Pipeline): img = LoadImage.convert_to_ndarray(input) if len(img.shape) == 2: img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) - img = img.astype(np.float) + img = img.astype(float) result = {'img': img} return result diff --git a/modelscope/pipelines/cv/tbs_detection_pipeline.py b/modelscope/pipelines/cv/tbs_detection_pipeline.py index 58831846..8bbac9c8 100644 --- a/modelscope/pipelines/cv/tbs_detection_pipeline.py +++ b/modelscope/pipelines/cv/tbs_detection_pipeline.py @@ -116,7 +116,7 @@ class TBSDetectionPipeline(Pipeline): - **labels** (`List[str]`, optional) -- The boxes's class_names of detected object in image. """ img = LoadImage.convert_to_ndarray(input) - img = img.astype(np.float) + img = img.astype(float) result = {'img': img, 'img_path': input} return result diff --git a/modelscope/utils/regress_test_utils.py b/modelscope/utils/regress_test_utils.py index 0f10c1ce..e03b3a7c 100644 --- a/modelscope/utils/regress_test_utils.py +++ b/modelscope/utils/regress_test_utils.py @@ -483,9 +483,9 @@ def numpify_tensor_nested(tensors, reduction=None, clip_value=10000): t = np.where(t > clip_value, clip_value, t) t = np.where(t < -clip_value, -clip_value, t) if reduction == 'sum': - return t.sum(dtype=np.float) + return t.sum(dtype=float) elif reduction == 'mean': - return t.mean(dtype=np.float) + return t.mean(dtype=float) return t return tensors diff --git a/modelscope/utils/test_utils.py b/modelscope/utils/test_utils.py index b4ce7299..03d293ec 100644 --- a/modelscope/utils/test_utils.py +++ b/modelscope/utils/test_utils.py @@ -150,7 +150,7 @@ def compare_arguments_nested(print_content, if arg1 is None: return True - elif isinstance(arg1, (int, str, bool, np.bool, np.integer, np.str)): + elif isinstance(arg1, (int, str, bool, np.bool_, np.integer, np.str_)): if arg1 != arg2: if print_content is not None: print(f'{print_content}, arg1:{arg1}, arg2:{arg2}') @@ -201,10 +201,8 @@ def compare_arguments_nested(print_content, return False return True elif isinstance(arg1, np.ndarray): - arg1 = np.where(np.equal(arg1, None), np.NaN, - arg1).astype(dtype=np.float) - arg2 = np.where(np.equal(arg2, None), np.NaN, - arg2).astype(dtype=np.float) + arg1 = np.where(np.equal(arg1, None), np.NaN, arg1).astype(dtype=float) + arg2 = np.where(np.equal(arg2, None), np.NaN, arg2).astype(dtype=float) if not all( np.isclose(arg1, arg2, rtol=rtol, atol=atol, equal_nan=True).flatten()): diff --git a/requirements/framework.txt b/requirements/framework.txt index e15e95eb..144cd31e 100644 --- a/requirements/framework.txt +++ b/requirements/framework.txt @@ -7,6 +7,8 @@ gast>=0.2.2 mmdet<=2.28.2 numpy<1.24.0 oss2 +# for datasets compatible +pandas<=1.5.3 Pillow>=6.2.0 # pyarrow 9.0.0 introduced event_loop core dump pyarrow>=6.0.0,!=9.0.0 From 038c5fea48d0d12b98de4e60fa3eef7b690abe19 Mon Sep 17 00:00:00 2001 From: "zhangzhicheng.zzc" Date: Mon, 8 May 2023 15:59:38 +0800 Subject: [PATCH 14/76] add verified if whl installed --- modelscope/utils/plugins.py | 144 ++++++++++++++++++++++++++++++++---- 1 file changed, 128 insertions(+), 16 deletions(-) diff --git a/modelscope/utils/plugins.py b/modelscope/utils/plugins.py index a83ca03c..9d238e7d 100644 --- a/modelscope/utils/plugins.py +++ b/modelscope/utils/plugins.py @@ -263,12 +263,11 @@ def import_module_and_submodules(package_name: str, def install_module_from_requirements(requirement_path, ): - """ + """ install module from requirements Args: requirement_path: The path of requirement file - Returns: - + No returns, raise error if failed """ install_list = [] @@ -292,6 +291,15 @@ def install_module_from_requirements(requirement_path, ): def import_module_from_file(module_name, file_path): + """ install module by name with file path + + Args: + module_name: the module name need to be import + file_path: the related file path that matched with the module name + + Returns: return the module class + + """ spec = importlib.util.spec_from_file_location(module_name, file_path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -299,6 +307,14 @@ def import_module_from_file(module_name, file_path): def import_module_from_model_dir(model_dir): + """ import all the necessary module from a model dir + + Args: + model_dir: model file location + + No returns, raise error if failed + + """ from pathlib import Path file_scanner = FilesAstScanning() file_scanner.traversal_files(model_dir) @@ -317,6 +333,14 @@ def import_module_from_model_dir(model_dir): def install_requirements_by_names(plugins: List[str]): + """ install the requirements by names + + Args: + plugins: name of plugins (pai-easyscv, transformers) + + No returns, raise error if failed + + """ plugins_manager = PluginsManager() uninstalled_plugins = [] for plugin in plugins: @@ -333,6 +357,14 @@ def install_requirements_by_names(plugins: List[str]): def install_requirements_by_files(requirements: List[str]): + """ install the requriements by files + + Args: + requirements: a list of files including requirements info (requirements.txt) + + No returns, raise error if failed + + """ for requirement in requirements: install_module_from_requirements(requirement) @@ -343,7 +375,8 @@ def register_plugins_repo(plugins: List[str]) -> None: install_requirements_by_names(plugins) modules = [] for plugin in plugins: - modules.extend(get_modules_from_package(plugin)) + module_name, module_version, _ = get_modules_from_package(plugin) + modules.extend(module_name) import_plugins(modules) @@ -362,12 +395,15 @@ DEFAULT_INDEX = 'https://pypi.org/simple/' def get_modules_from_package(package): - """ to get the modules from a installed package + """ to get the modules from an installed package Args: package: The distribution name or package name Returns: + import_names: The modules that in the package distribution + import_version: The version of those modules, should be same and identical + package_name: The package name, if installed by whl file, the package is unknown, should be passed """ from zipfile import ZipFile @@ -378,8 +414,6 @@ def get_modules_from_package(package): from urllib.parse import urlparse from urllib import request as urllib2 from pip._internal.utils.packaging import get_requirement - req = get_requirement(package) - package = req.name def urlretrieve(url, filename, data=None, auth=None): if auth is not None: @@ -591,24 +625,58 @@ def get_modules_from_package(package): return result def discover_import_names(whl_file): + import re logger.debug('finding import names') zipfile = ZipFile(file=whl_file) namelist = zipfile.namelist() [top_level_fname ] = [x for x in namelist if x.endswith('top_level.txt')] + [metadata_fname + ] = [x for x in namelist if x.endswith('.dist-info/METADATA')] all_names = zipfile.read(top_level_fname).decode( 'utf-8').strip().splitlines() + metadata = zipfile.read(metadata_fname).decode('utf-8') public_names = [n for n in all_names if not n.startswith('_')] - return public_names + + version_pattern = re.compile(r'^Version: (?P.+)$', + re.MULTILINE) + name_pattern = re.compile(r'^Name: (?P.+)$', re.MULTILINE) + + version_match = version_pattern.search(metadata) + name_match = name_pattern.search(metadata) + + module_version = version_match.group('version') + module_name = name_match.group('name') + + return public_names, module_version, module_name tmpdir = mkdtemp() - data = get(package, tmpdir=tmpdir) - import_names = discover_import_names(data['path']) + if package.endswith('.whl'): + """if user using .whl file then parse the whl to get the module name""" + if not os.path.isfile(package): + file_name = os.path.basename(package) + file_path = os.path.join(tmpdir, file_name) + whl_file, _ = _download_dist(package, file_path, None, None) + else: + whl_file = package + else: + """if user using package name then generate whl file and parse the file to get the module name by + the discover_import_names method + """ + req = get_requirement(package) + package = req.name + data = get(package, tmpdir=tmpdir) + whl_file = data['path'] + import_names, import_version, package_name = discover_import_names( + whl_file) shutil.rmtree(tmpdir) - return import_names + return import_names, import_version, package_name class PluginsManager(object): + """ + plugins manager class + """ def __init__(self, cache_dir=MODELSCOPE_FILE_DIR, @@ -633,12 +701,26 @@ class PluginsManager(object): package: the package name need to be installed Returns: + if_installed: True if installed + version: the version of installed or None if not installed """ if package.split('.')[-1] == 'whl': - return False, '' + # install from whl should test package name instead of module name + _, module_version, package_name = get_modules_from_package(package) + local_installed, version = PluginsManager._check_plugin_installed( + package_name) + if local_installed and module_version != version: + return False, version + elif not local_installed: + return False, version + return True, module_version + else: + return PluginsManager._check_plugin_installed(package) + @staticmethod + def _check_plugin_installed(package, verified_version=None): from pip._internal.utils.packaging import get_requirement, specifiers req = get_requirement(package) @@ -656,11 +738,15 @@ class PluginsManager(object): if not installed_valid_version: installed = False break + except KeyError: version = '' installed = False - return installed, version + if installed and verified_version is not None and verified_version != version: + return False, verified_version + else: + return installed, version @staticmethod def pip_command( @@ -675,6 +761,9 @@ class PluginsManager(object): such as ['-r', 'requirements'] Returns: + status_code: The pip command status code, 0 if success, else is failed + options: parsed option from system args by pip command + args: the unknown args that could be parsed by pip command """ from pip._internal.commands import create_command @@ -702,6 +791,7 @@ class PluginsManager(object): Args: install_args (list): List of arguments passed to `pip install`. index_url (str, optional): The pypi index url. + force_update: If force update on or off """ if len(install_args) == 0: @@ -730,6 +820,16 @@ class PluginsManager(object): return status_code, install_args def parse_args_info(self, args: List[str], options): + """ + parse arguments input info + Args: + args: the list of args from pip command output + options: the options that parsed from system args by pip command method + + Returns: + installed_package: generate installed package info in order to store in the file + the info includes: name, url and desc of the package + """ installed_package = [] # the case of install with requirements @@ -781,6 +881,15 @@ class PluginsManager(object): def uninstall_plugins(self, uninstall_args: Union[str, List], is_yes=False): + """ + uninstall plugins + Args: + uninstall_args: args used to uninstall by pip command + is_yes: force yes without verified + + Returns: status code, and uninstall args + + """ if is_yes is not None: uninstall_args += ['-y'] @@ -862,6 +971,7 @@ class PluginsManager(object): show_all: show installed and official supported if True, else only those installed Returns: + local_plugins_info: show the list of plugins info """ local_plugins_info = self._get_plugins_from_file() @@ -901,6 +1011,7 @@ class PluginsManager(object): override: Override the file by the list if True, else only update. Returns: + local_plugins_info_json: the json version of updated plugins info """ local_plugins_info = self._get_plugins_from_file() @@ -921,12 +1032,12 @@ class PluginsManager(object): self, package_names: Union[str, list], ): - """ - + """remove the plugins from file Args: package_names: package name Returns: + local_plugins_info_json: the json version of updated plugins info """ local_plugins_info = self._get_plugins_from_file() @@ -1012,4 +1123,5 @@ class EnvsManager(object): if __name__ == '__main__': install_requirements_by_files(['adaseq']) - import_name = get_modules_from_package('pai-easycv') + import_name, import_version, package_name = get_modules_from_package( + 'pai-easycv') From 46072898dab861bb6758eb1d7f5cbcfe9369325e Mon Sep 17 00:00:00 2001 From: "jiangnana.jnn" Date: Tue, 9 May 2023 17:58:01 +0800 Subject: [PATCH 15/76] remove easycv codes, plugin access Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/11965727 * remove easycv codes * fix custome msdatasets import and remove metainfo * fix pipeline imports * fix pre-check * fix models import * fix pre-check * merge master --- modelscope/metainfo.py | 21 -- modelscope/models/cv/__init__.py | 5 +- modelscope/models/cv/easycv_base.py | 25 -- .../models/cv/face_2d_keypoints/__init__.py | 20 -- .../face_2d_keypoints_align.py | 16 -- .../models/cv/hand_2d_keypoints/__init__.py | 20 -- .../cv/hand_2d_keypoints/hand_2d_keypoints.py | 16 -- .../cv/human_wholebody_keypoint/__init__.py | 22 -- .../human_wholebody_keypoint.py | 17 -- .../image_panoptic_segmentation/__init__.py | 1 - .../r50_panseg_model.py | 18 -- .../image_semantic_segmentation/segformer.py | 16 -- modelscope/models/cv/object_detection/dino.py | 16 -- .../models/cv/object_detection/yolox_pai.py | 21 -- .../dataset_cls/custom_datasets/__init__.py | 12 - .../face_2d_keypoins/__init__.py | 20 -- .../face_2d_keypoints_dataset.py | 38 --- .../hand_2d_keypoints/__init__.py | 22 -- .../hand_2d_keypoints_dataset.py | 39 --- .../human_wholebody_keypoint/__init__.py | 22 -- .../human_wholebody_keypoint_dataset.py | 40 --- .../image_classification/__init__.py | 20 -- .../classification_dataset.py | 38 --- .../image_semantic_segmentation/__init__.py | 20 -- .../segmentation_dataset.py | 43 --- .../object_detection/__init__.py | 22 -- .../object_detection/detection_dataset.py | 98 ------- modelscope/pipelines/cv/__init__.py | 29 +-- .../pipelines/cv/easycv_pipelines/__init__.py | 28 -- .../pipelines/cv/easycv_pipelines/base.py | 123 --------- .../cv/easycv_pipelines/detection_pipeline.py | 66 ----- .../face_2d_keypoints_pipeline.py | 244 ------------------ .../human_wholebody_keypoint_pipeline.py | 67 ----- .../easycv_pipelines/segmentation_pipeline.py | 47 ---- .../cv/hand_2d_keypoints_pipeline.py | 51 ---- .../image_panoptic_segmentation_pipeline.py | 135 ---------- modelscope/trainers/easycv/__init__.py | 19 -- modelscope/trainers/easycv/trainer.py | 183 ------------- modelscope/trainers/easycv/utils/__init__.py | 21 -- modelscope/trainers/easycv/utils/hooks.py | 29 --- modelscope/trainers/easycv/utils/metric.py | 62 ----- .../trainers/easycv/utils/register_util.py | 97 ------- tests/msdatasets/test_ms_dataset.py | 11 - tests/pipelines/easycv_pipelines/__init__.py | 0 .../test_panoptic_segmentation_pipeline.py | 36 --- .../test_segmentation_pipeline.py | 88 ------- tests/trainers/easycv/__init__.py | 0 tests/trainers/easycv/test_easycv_trainer.py | 238 ----------------- .../test_easycv_trainer_detection_dino.py | 69 ----- .../test_easycv_trainer_face_2d_keypoints.py | 72 ------ .../test_easycv_trainer_hand_2d_keypoints.py | 72 ------ .../test_easycv_trainer_hand_detection.py | 63 ----- ...est_easycv_trainer_panoptic_mask2former.py | 70 ----- ...asycv_trainer_realtime_object_detection.py | 99 ------- tests/trainers/easycv/test_segformer.py | 72 ------ tests/trainers/model_trainer_map.py | 35 +-- 56 files changed, 15 insertions(+), 2779 deletions(-) delete mode 100644 modelscope/models/cv/easycv_base.py delete mode 100644 modelscope/models/cv/face_2d_keypoints/__init__.py delete mode 100644 modelscope/models/cv/face_2d_keypoints/face_2d_keypoints_align.py delete mode 100644 modelscope/models/cv/hand_2d_keypoints/__init__.py delete mode 100644 modelscope/models/cv/hand_2d_keypoints/hand_2d_keypoints.py delete mode 100644 modelscope/models/cv/human_wholebody_keypoint/__init__.py delete mode 100644 modelscope/models/cv/human_wholebody_keypoint/human_wholebody_keypoint.py delete mode 100644 modelscope/models/cv/image_panoptic_segmentation/r50_panseg_model.py delete mode 100644 modelscope/models/cv/image_semantic_segmentation/segformer.py delete mode 100644 modelscope/models/cv/object_detection/dino.py delete mode 100644 modelscope/models/cv/object_detection/yolox_pai.py delete mode 100644 modelscope/msdatasets/dataset_cls/custom_datasets/face_2d_keypoins/__init__.py delete mode 100644 modelscope/msdatasets/dataset_cls/custom_datasets/face_2d_keypoins/face_2d_keypoints_dataset.py delete mode 100644 modelscope/msdatasets/dataset_cls/custom_datasets/hand_2d_keypoints/__init__.py delete mode 100644 modelscope/msdatasets/dataset_cls/custom_datasets/hand_2d_keypoints/hand_2d_keypoints_dataset.py delete mode 100644 modelscope/msdatasets/dataset_cls/custom_datasets/human_wholebody_keypoint/__init__.py delete mode 100644 modelscope/msdatasets/dataset_cls/custom_datasets/human_wholebody_keypoint/human_wholebody_keypoint_dataset.py delete mode 100644 modelscope/msdatasets/dataset_cls/custom_datasets/image_classification/__init__.py delete mode 100644 modelscope/msdatasets/dataset_cls/custom_datasets/image_classification/classification_dataset.py delete mode 100644 modelscope/msdatasets/dataset_cls/custom_datasets/image_semantic_segmentation/__init__.py delete mode 100644 modelscope/msdatasets/dataset_cls/custom_datasets/image_semantic_segmentation/segmentation_dataset.py delete mode 100644 modelscope/msdatasets/dataset_cls/custom_datasets/object_detection/__init__.py delete mode 100644 modelscope/msdatasets/dataset_cls/custom_datasets/object_detection/detection_dataset.py delete mode 100644 modelscope/pipelines/cv/easycv_pipelines/__init__.py delete mode 100644 modelscope/pipelines/cv/easycv_pipelines/base.py delete mode 100644 modelscope/pipelines/cv/easycv_pipelines/detection_pipeline.py delete mode 100644 modelscope/pipelines/cv/easycv_pipelines/face_2d_keypoints_pipeline.py delete mode 100644 modelscope/pipelines/cv/easycv_pipelines/human_wholebody_keypoint_pipeline.py delete mode 100644 modelscope/pipelines/cv/easycv_pipelines/segmentation_pipeline.py delete mode 100644 modelscope/pipelines/cv/hand_2d_keypoints_pipeline.py delete mode 100644 modelscope/pipelines/cv/image_panoptic_segmentation_pipeline.py delete mode 100644 modelscope/trainers/easycv/__init__.py delete mode 100644 modelscope/trainers/easycv/trainer.py delete mode 100644 modelscope/trainers/easycv/utils/__init__.py delete mode 100644 modelscope/trainers/easycv/utils/hooks.py delete mode 100644 modelscope/trainers/easycv/utils/metric.py delete mode 100644 modelscope/trainers/easycv/utils/register_util.py delete mode 100644 tests/pipelines/easycv_pipelines/__init__.py delete mode 100644 tests/pipelines/easycv_pipelines/test_panoptic_segmentation_pipeline.py delete mode 100644 tests/pipelines/easycv_pipelines/test_segmentation_pipeline.py delete mode 100644 tests/trainers/easycv/__init__.py delete mode 100644 tests/trainers/easycv/test_easycv_trainer.py delete mode 100644 tests/trainers/easycv/test_easycv_trainer_detection_dino.py delete mode 100644 tests/trainers/easycv/test_easycv_trainer_face_2d_keypoints.py delete mode 100644 tests/trainers/easycv/test_easycv_trainer_hand_2d_keypoints.py delete mode 100644 tests/trainers/easycv/test_easycv_trainer_hand_detection.py delete mode 100644 tests/trainers/easycv/test_easycv_trainer_panoptic_mask2former.py delete mode 100644 tests/trainers/easycv/test_easycv_trainer_realtime_object_detection.py delete mode 100644 tests/trainers/easycv/test_segformer.py diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index 432f7dd8..8d6034e2 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -118,13 +118,6 @@ class Models(object): longshortnet = 'longshortnet' pedestrian_attribute_recognition = 'pedestrian-attribute-recognition' - # EasyCV models - yolox = 'YOLOX' - segformer = 'Segformer' - hand_2d_keypoints = 'HRNet-Hand2D-Keypoints' - image_object_detection_auto = 'image-object-detection-auto' - dino = 'DINO' - # nlp models bert = 'bert' palm = 'palm-v2' @@ -279,8 +272,6 @@ class Pipelines(object): tbs_detection = 'tbs-detection' object_detection = 'vit-object-detection' abnormal_object_detection = 'abnormal-object-detection' - easycv_detection = 'easycv-detection' - easycv_segmentation = 'easycv-segmentation' face_2d_keypoints = 'mobilenet_face-2d-keypoints_alignment' salient_detection = 'u2net-salient-detection' salient_boudary_detection = 'res2net-salient-detection' @@ -349,7 +340,6 @@ class Pipelines(object): video_single_object_tracking_procontext = 'procontext-vitb-video-single-object-tracking' video_multi_object_tracking = 'video-multi-object-tracking' image_panoptic_segmentation = 'image-panoptic-segmentation' - image_panoptic_segmentation_easycv = 'image-panoptic-segmentation-easycv' video_summarization = 'googlenet_pgl_video_summarization' language_guided_video_summarization = 'clip-it-video-summarization' image_semantic_segmentation = 'image-semantic-segmentation' @@ -914,7 +904,6 @@ class Trainers(CVTrainers, NLPTrainers, MultiModalTrainers, AudioTrainers): """ default = 'trainer' - easycv = 'easycv' tinynas_damoyolo = 'tinynas-damoyolo' @staticmethod @@ -936,8 +925,6 @@ class Trainers(CVTrainers, NLPTrainers, MultiModalTrainers, AudioTrainers): return Fields.multi_modal elif attribute_or_value == Trainers.default: return Trainers.default - elif attribute_or_value == Trainers.easycv: - return Trainers.easycv else: return 'unknown' @@ -1168,14 +1155,6 @@ class LR_Schedulers(object): class CustomDatasets(object): """ Names for different datasets. """ - ClsDataset = 'ClsDataset' - Face2dKeypointsDataset = 'FaceKeypointDataset' - HandCocoWholeBodyDataset = 'HandCocoWholeBodyDataset' - HumanWholeBodyKeypointDataset = 'WholeBodyCocoTopDownDataset' - SegDataset = 'SegDataset' - DetDataset = 'DetDataset' - DetImagesMixDataset = 'DetImagesMixDataset' - PanopticDataset = 'PanopticDataset' PairedDataset = 'PairedDataset' SiddDataset = 'SiddDataset' GoproDataset = 'GoproDataset' diff --git a/modelscope/models/cv/__init__.py b/modelscope/models/cv/__init__.py index 21487216..39acec69 100644 --- a/modelscope/models/cv/__init__.py +++ b/modelscope/models/cv/__init__.py @@ -4,9 +4,8 @@ from . import (action_recognition, animal_recognition, bad_image_detecting, body_2d_keypoints, body_3d_keypoints, cartoon, cmdssl_video_embedding, controllable_image_generation, - crowd_counting, face_2d_keypoints, face_detection, - face_generation, face_reconstruction, human_reconstruction, - human_wholebody_keypoint, image_classification, + crowd_counting, face_detection, face_generation, + face_reconstruction, human_reconstruction, image_classification, image_color_enhance, image_colorization, image_defrcn_fewshot, image_denoise, image_inpainting, image_instance_segmentation, image_matching, image_mvs_depth_estimation, diff --git a/modelscope/models/cv/easycv_base.py b/modelscope/models/cv/easycv_base.py deleted file mode 100644 index 7bc35e84..00000000 --- a/modelscope/models/cv/easycv_base.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from easycv.models.base import BaseModel -from easycv.utils.ms_utils import EasyCVMeta - -from modelscope.models.base import TorchModel - - -class EasyCVBaseModel(BaseModel, TorchModel): - """Base model for EasyCV.""" - - def __init__(self, model_dir=None, args=(), kwargs={}): - kwargs.pop(EasyCVMeta.ARCH, None) # pop useless keys - BaseModel.__init__(self) - TorchModel.__init__(self, model_dir=model_dir) - - def forward(self, img, mode='train', **kwargs): - if self.training: - losses = self.forward_train(img, **kwargs) - loss, log_vars = self._parse_losses(losses) - return dict(loss=loss, log_vars=log_vars) - else: - return self.forward_test(img, **kwargs) - - def __call__(self, *args, **kwargs): - return self.forward(*args, **kwargs) diff --git a/modelscope/models/cv/face_2d_keypoints/__init__.py b/modelscope/models/cv/face_2d_keypoints/__init__.py deleted file mode 100644 index 636ba0f4..00000000 --- a/modelscope/models/cv/face_2d_keypoints/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from typing import TYPE_CHECKING - -from modelscope.utils.import_utils import LazyImportModule - -if TYPE_CHECKING: - from .face_2d_keypoints_align import Face2DKeypoints - -else: - _import_structure = {'face_2d_keypoints_align': ['Face2DKeypoints']} - - import sys - - sys.modules[__name__] = LazyImportModule( - __name__, - globals()['__file__'], - _import_structure, - module_spec=__spec__, - extra_objects={}, - ) diff --git a/modelscope/models/cv/face_2d_keypoints/face_2d_keypoints_align.py b/modelscope/models/cv/face_2d_keypoints/face_2d_keypoints_align.py deleted file mode 100644 index 468662a0..00000000 --- a/modelscope/models/cv/face_2d_keypoints/face_2d_keypoints_align.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from easycv.models.face.face_keypoint import FaceKeypoint - -from modelscope.metainfo import Models -from modelscope.models.builder import MODELS -from modelscope.models.cv.easycv_base import EasyCVBaseModel -from modelscope.utils.constant import Tasks - - -@MODELS.register_module( - group_key=Tasks.face_2d_keypoints, module_name=Models.face_2d_keypoints) -class Face2DKeypoints(EasyCVBaseModel, FaceKeypoint): - - def __init__(self, model_dir=None, *args, **kwargs): - EasyCVBaseModel.__init__(self, model_dir, args, kwargs) - FaceKeypoint.__init__(self, *args, **kwargs) diff --git a/modelscope/models/cv/hand_2d_keypoints/__init__.py b/modelscope/models/cv/hand_2d_keypoints/__init__.py deleted file mode 100644 index 2b06f19a..00000000 --- a/modelscope/models/cv/hand_2d_keypoints/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from typing import TYPE_CHECKING - -from modelscope.utils.import_utils import LazyImportModule - -if TYPE_CHECKING: - from .hand_2d_keypoints import Hand2dKeyPoints - -else: - _import_structure = {'hand_2d_keypoints': ['Hand2dKeyPoints']} - - import sys - - sys.modules[__name__] = LazyImportModule( - __name__, - globals()['__file__'], - _import_structure, - module_spec=__spec__, - extra_objects={}, - ) diff --git a/modelscope/models/cv/hand_2d_keypoints/hand_2d_keypoints.py b/modelscope/models/cv/hand_2d_keypoints/hand_2d_keypoints.py deleted file mode 100644 index 15a97c30..00000000 --- a/modelscope/models/cv/hand_2d_keypoints/hand_2d_keypoints.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from easycv.models.pose import TopDown - -from modelscope.metainfo import Models -from modelscope.models.builder import MODELS -from modelscope.models.cv.easycv_base import EasyCVBaseModel -from modelscope.utils.constant import Tasks - - -@MODELS.register_module( - group_key=Tasks.hand_2d_keypoints, module_name=Models.hand_2d_keypoints) -class Hand2dKeyPoints(EasyCVBaseModel, TopDown): - - def __init__(self, model_dir=None, *args, **kwargs): - EasyCVBaseModel.__init__(self, model_dir, args, kwargs) - TopDown.__init__(self, *args, **kwargs) diff --git a/modelscope/models/cv/human_wholebody_keypoint/__init__.py b/modelscope/models/cv/human_wholebody_keypoint/__init__.py deleted file mode 100644 index 30e23457..00000000 --- a/modelscope/models/cv/human_wholebody_keypoint/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from typing import TYPE_CHECKING - -from modelscope.utils.import_utils import LazyImportModule - -if TYPE_CHECKING: - from .human_wholebody_keypoint import HumanWholeBodyKeypoint - -else: - _import_structure = { - 'human_wholebody_keypoint': ['HumanWholeBodyKeypoint'] - } - - import sys - - sys.modules[__name__] = LazyImportModule( - __name__, - globals()['__file__'], - _import_structure, - module_spec=__spec__, - extra_objects={}, - ) diff --git a/modelscope/models/cv/human_wholebody_keypoint/human_wholebody_keypoint.py b/modelscope/models/cv/human_wholebody_keypoint/human_wholebody_keypoint.py deleted file mode 100644 index dd3c0290..00000000 --- a/modelscope/models/cv/human_wholebody_keypoint/human_wholebody_keypoint.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from easycv.models.pose.top_down import TopDown - -from modelscope.metainfo import Models -from modelscope.models.builder import MODELS -from modelscope.models.cv.easycv_base import EasyCVBaseModel -from modelscope.utils.constant import Tasks - - -@MODELS.register_module( - group_key=Tasks.human_wholebody_keypoint, - module_name=Models.human_wholebody_keypoint) -class HumanWholeBodyKeypoint(EasyCVBaseModel, TopDown): - - def __init__(self, model_dir=None, *args, **kwargs): - EasyCVBaseModel.__init__(self, model_dir, args, kwargs) - TopDown.__init__(self, *args, **kwargs) diff --git a/modelscope/models/cv/image_panoptic_segmentation/__init__.py b/modelscope/models/cv/image_panoptic_segmentation/__init__.py index 1af5b6f8..2b2be4b7 100644 --- a/modelscope/models/cv/image_panoptic_segmentation/__init__.py +++ b/modelscope/models/cv/image_panoptic_segmentation/__init__.py @@ -5,7 +5,6 @@ from modelscope.utils.import_utils import LazyImportModule if TYPE_CHECKING: from .panseg_model import SwinLPanopticSegmentation - from .r50_panseg_model import R50PanopticSegmentation else: _import_structure = { diff --git a/modelscope/models/cv/image_panoptic_segmentation/r50_panseg_model.py b/modelscope/models/cv/image_panoptic_segmentation/r50_panseg_model.py deleted file mode 100644 index 73b6b76c..00000000 --- a/modelscope/models/cv/image_panoptic_segmentation/r50_panseg_model.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. - -from easycv.models.segmentation import Mask2Former - -from modelscope.metainfo import Models -from modelscope.models.builder import MODELS -from modelscope.models.cv.easycv_base import EasyCVBaseModel -from modelscope.utils.constant import Tasks - - -@MODELS.register_module( - group_key=Tasks.image_segmentation, - module_name=Models.r50_panoptic_segmentation) -class R50PanopticSegmentation(EasyCVBaseModel, Mask2Former): - - def __init__(self, model_dir=None, *args, **kwargs): - EasyCVBaseModel.__init__(self, model_dir, args, kwargs) - Mask2Former.__init__(self, *args, **kwargs) diff --git a/modelscope/models/cv/image_semantic_segmentation/segformer.py b/modelscope/models/cv/image_semantic_segmentation/segformer.py deleted file mode 100644 index 46303526..00000000 --- a/modelscope/models/cv/image_semantic_segmentation/segformer.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from easycv.models.segmentation import EncoderDecoder - -from modelscope.metainfo import Models -from modelscope.models.builder import MODELS -from modelscope.models.cv.easycv_base import EasyCVBaseModel -from modelscope.utils.constant import Tasks - - -@MODELS.register_module( - group_key=Tasks.image_segmentation, module_name=Models.segformer) -class Segformer(EasyCVBaseModel, EncoderDecoder): - - def __init__(self, model_dir=None, *args, **kwargs): - EasyCVBaseModel.__init__(self, model_dir, args, kwargs) - EncoderDecoder.__init__(self, *args, **kwargs) diff --git a/modelscope/models/cv/object_detection/dino.py b/modelscope/models/cv/object_detection/dino.py deleted file mode 100644 index e6c652f1..00000000 --- a/modelscope/models/cv/object_detection/dino.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from easycv.models.detection.detectors import Detection as _Detection - -from modelscope.metainfo import Models -from modelscope.models.builder import MODELS -from modelscope.models.cv.easycv_base import EasyCVBaseModel -from modelscope.utils.constant import Tasks - - -@MODELS.register_module( - group_key=Tasks.image_object_detection, module_name=Models.dino) -class DINO(EasyCVBaseModel, _Detection): - - def __init__(self, model_dir=None, *args, **kwargs): - EasyCVBaseModel.__init__(self, model_dir, args, kwargs) - _Detection.__init__(self, *args, **kwargs) diff --git a/modelscope/models/cv/object_detection/yolox_pai.py b/modelscope/models/cv/object_detection/yolox_pai.py deleted file mode 100644 index 7888cf82..00000000 --- a/modelscope/models/cv/object_detection/yolox_pai.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from easycv.models.detection.detectors import YOLOX as _YOLOX - -from modelscope.metainfo import Models -from modelscope.models.builder import MODELS -from modelscope.models.cv.easycv_base import EasyCVBaseModel -from modelscope.utils.constant import Tasks - - -@MODELS.register_module( - group_key=Tasks.image_object_detection, module_name=Models.yolox) -@MODELS.register_module( - group_key=Tasks.image_object_detection, - module_name=Models.image_object_detection_auto) -@MODELS.register_module( - group_key=Tasks.domain_specific_object_detection, module_name=Models.yolox) -class YOLOX(EasyCVBaseModel, _YOLOX): - - def __init__(self, model_dir=None, *args, **kwargs): - EasyCVBaseModel.__init__(self, model_dir, args, kwargs) - _YOLOX.__init__(self, *args, **kwargs) diff --git a/modelscope/msdatasets/dataset_cls/custom_datasets/__init__.py b/modelscope/msdatasets/dataset_cls/custom_datasets/__init__.py index 9eb62168..a367fe79 100644 --- a/modelscope/msdatasets/dataset_cls/custom_datasets/__init__.py +++ b/modelscope/msdatasets/dataset_cls/custom_datasets/__init__.py @@ -27,12 +27,6 @@ if TYPE_CHECKING: from .video_frame_interpolation import VideoFrameInterpolationDataset from .video_stabilization import VideoStabilizationDataset from .video_super_resolution import VideoSuperResolutionDataset - from .image_semantic_segmentation import SegDataset - from .face_2d_keypoins import FaceKeypointDataset - from .hand_2d_keypoints import HandCocoWholeBodyDataset - from .human_wholebody_keypoint import WholeBodyCocoTopDownDataset - from .image_classification import ClsDataset - from .object_detection import DetDataset, DetImagesMixDataset from .ocr_detection import DataLoader, ImageDataset, QuadMeasurer from .ocr_recognition_dataset import OCRRecognitionDataset from .image_colorization import ImageColorizationDataset @@ -66,12 +60,6 @@ else: 'video_frame_interpolation': ['VideoFrameInterpolationDataset'], 'video_stabilization': ['VideoStabilizationDataset'], 'video_super_resolution': ['VideoSuperResolutionDataset'], - 'image_semantic_segmentation': ['SegDataset'], - 'face_2d_keypoins': ['FaceKeypointDataset'], - 'hand_2d_keypoints': ['HandCocoWholeBodyDataset'], - 'human_wholebody_keypoint': ['WholeBodyCocoTopDownDataset'], - 'image_classification': ['ClsDataset'], - 'object_detection': ['DetDataset', 'DetImagesMixDataset'], 'ocr_detection': ['DataLoader', 'ImageDataset', 'QuadMeasurer'], 'ocr_recognition_dataset': ['OCRRecognitionDataset'], 'image_colorization': ['ImageColorizationDataset'], diff --git a/modelscope/msdatasets/dataset_cls/custom_datasets/face_2d_keypoins/__init__.py b/modelscope/msdatasets/dataset_cls/custom_datasets/face_2d_keypoins/__init__.py deleted file mode 100644 index e9d76b7e..00000000 --- a/modelscope/msdatasets/dataset_cls/custom_datasets/face_2d_keypoins/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from typing import TYPE_CHECKING - -from modelscope.utils.import_utils import LazyImportModule - -if TYPE_CHECKING: - from .face_2d_keypoints_dataset import FaceKeypointDataset - -else: - _import_structure = {'face_2d_keypoints_dataset': ['FaceKeypointDataset']} - - import sys - - sys.modules[__name__] = LazyImportModule( - __name__, - globals()['__file__'], - _import_structure, - module_spec=__spec__, - extra_objects={}, - ) diff --git a/modelscope/msdatasets/dataset_cls/custom_datasets/face_2d_keypoins/face_2d_keypoints_dataset.py b/modelscope/msdatasets/dataset_cls/custom_datasets/face_2d_keypoins/face_2d_keypoints_dataset.py deleted file mode 100644 index 9f55901f..00000000 --- a/modelscope/msdatasets/dataset_cls/custom_datasets/face_2d_keypoins/face_2d_keypoints_dataset.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from easycv.datasets.face import FaceKeypointDataset as _FaceKeypointDataset - -from modelscope.metainfo import CustomDatasets -from modelscope.msdatasets.dataset_cls.custom_datasets import CUSTOM_DATASETS -from modelscope.msdatasets.dataset_cls.custom_datasets.easycv_base import \ - EasyCVBaseDataset -from modelscope.utils.constant import Tasks - - -@CUSTOM_DATASETS.register_module( - group_key=Tasks.face_2d_keypoints, - module_name=CustomDatasets.Face2dKeypointsDataset) -class FaceKeypointDataset(EasyCVBaseDataset, _FaceKeypointDataset): - """EasyCV dataset for face 2d keypoints. - - Args: - split_config (dict): Dataset root path from MSDataset, e.g. - {"train":"local cache path"} or {"evaluation":"local cache path"}. - preprocessor (Preprocessor): An optional preprocessor instance, please make sure the preprocessor fits for - the model if supplied. Not support yet. - mode: Training or Evaluation. - """ - - def __init__(self, - split_config=None, - preprocessor=None, - mode=None, - *args, - **kwargs) -> None: - EasyCVBaseDataset.__init__( - self, - split_config=split_config, - preprocessor=preprocessor, - mode=mode, - args=args, - kwargs=kwargs) - _FaceKeypointDataset.__init__(self, *args, **kwargs) diff --git a/modelscope/msdatasets/dataset_cls/custom_datasets/hand_2d_keypoints/__init__.py b/modelscope/msdatasets/dataset_cls/custom_datasets/hand_2d_keypoints/__init__.py deleted file mode 100644 index 3af670e3..00000000 --- a/modelscope/msdatasets/dataset_cls/custom_datasets/hand_2d_keypoints/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from typing import TYPE_CHECKING - -from modelscope.utils.import_utils import LazyImportModule - -if TYPE_CHECKING: - from .hand_2d_keypoints_dataset import HandCocoWholeBodyDataset - -else: - _import_structure = { - 'hand_2d_keypoints_dataset': ['HandCocoWholeBodyDataset'] - } - - import sys - - sys.modules[__name__] = LazyImportModule( - __name__, - globals()['__file__'], - _import_structure, - module_spec=__spec__, - extra_objects={}, - ) diff --git a/modelscope/msdatasets/dataset_cls/custom_datasets/hand_2d_keypoints/hand_2d_keypoints_dataset.py b/modelscope/msdatasets/dataset_cls/custom_datasets/hand_2d_keypoints/hand_2d_keypoints_dataset.py deleted file mode 100644 index c6163715..00000000 --- a/modelscope/msdatasets/dataset_cls/custom_datasets/hand_2d_keypoints/hand_2d_keypoints_dataset.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from easycv.datasets.pose import \ - HandCocoWholeBodyDataset as _HandCocoWholeBodyDataset - -from modelscope.metainfo import CustomDatasets -from modelscope.msdatasets.dataset_cls.custom_datasets import CUSTOM_DATASETS -from modelscope.msdatasets.dataset_cls.custom_datasets.easycv_base import \ - EasyCVBaseDataset -from modelscope.utils.constant import Tasks - - -@CUSTOM_DATASETS.register_module( - group_key=Tasks.hand_2d_keypoints, - module_name=CustomDatasets.HandCocoWholeBodyDataset) -class HandCocoWholeBodyDataset(EasyCVBaseDataset, _HandCocoWholeBodyDataset): - """EasyCV dataset for human hand 2d keypoints. - - Args: - split_config (dict): Dataset root path from MSDataset, e.g. - {"train":"local cache path"} or {"evaluation":"local cache path"}. - preprocessor (Preprocessor): An optional preprocessor instance, please make sure the preprocessor fits for - the model if supplied. Not support yet. - mode: Training or Evaluation. - """ - - def __init__(self, - split_config=None, - preprocessor=None, - mode=None, - *args, - **kwargs) -> None: - EasyCVBaseDataset.__init__( - self, - split_config=split_config, - preprocessor=preprocessor, - mode=mode, - args=args, - kwargs=kwargs) - _HandCocoWholeBodyDataset.__init__(self, *args, **kwargs) diff --git a/modelscope/msdatasets/dataset_cls/custom_datasets/human_wholebody_keypoint/__init__.py b/modelscope/msdatasets/dataset_cls/custom_datasets/human_wholebody_keypoint/__init__.py deleted file mode 100644 index 472ed2d8..00000000 --- a/modelscope/msdatasets/dataset_cls/custom_datasets/human_wholebody_keypoint/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from typing import TYPE_CHECKING - -from modelscope.utils.import_utils import LazyImportModule - -if TYPE_CHECKING: - from .human_wholebody_keypoint_dataset import WholeBodyCocoTopDownDataset - -else: - _import_structure = { - 'human_wholebody_keypoint_dataset': ['WholeBodyCocoTopDownDataset'] - } - - import sys - - sys.modules[__name__] = LazyImportModule( - __name__, - globals()['__file__'], - _import_structure, - module_spec=__spec__, - extra_objects={}, - ) diff --git a/modelscope/msdatasets/dataset_cls/custom_datasets/human_wholebody_keypoint/human_wholebody_keypoint_dataset.py b/modelscope/msdatasets/dataset_cls/custom_datasets/human_wholebody_keypoint/human_wholebody_keypoint_dataset.py deleted file mode 100644 index 59c97af8..00000000 --- a/modelscope/msdatasets/dataset_cls/custom_datasets/human_wholebody_keypoint/human_wholebody_keypoint_dataset.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from easycv.datasets.pose import \ - WholeBodyCocoTopDownDataset as _WholeBodyCocoTopDownDataset - -from modelscope.metainfo import CustomDatasets -from modelscope.msdatasets.dataset_cls.custom_datasets import CUSTOM_DATASETS -from modelscope.msdatasets.dataset_cls.custom_datasets.easycv_base import \ - EasyCVBaseDataset -from modelscope.utils.constant import Tasks - - -@CUSTOM_DATASETS.register_module( - group_key=Tasks.human_wholebody_keypoint, - module_name=CustomDatasets.HumanWholeBodyKeypointDataset) -class WholeBodyCocoTopDownDataset(EasyCVBaseDataset, - _WholeBodyCocoTopDownDataset): - """EasyCV dataset for human whole body 2d keypoints. - - Args: - split_config (dict): Dataset root path from MSDataset, e.g. - {"train":"local cache path"} or {"evaluation":"local cache path"}. - preprocessor (Preprocessor): An optional preprocessor instance, please make sure the preprocessor fits for - the model if supplied. Not support yet. - mode: Training or Evaluation. - """ - - def __init__(self, - split_config=None, - preprocessor=None, - mode=None, - *args, - **kwargs) -> None: - EasyCVBaseDataset.__init__( - self, - split_config=split_config, - preprocessor=preprocessor, - mode=mode, - args=args, - kwargs=kwargs) - _WholeBodyCocoTopDownDataset.__init__(self, *args, **kwargs) diff --git a/modelscope/msdatasets/dataset_cls/custom_datasets/image_classification/__init__.py b/modelscope/msdatasets/dataset_cls/custom_datasets/image_classification/__init__.py deleted file mode 100644 index 95e8d7a1..00000000 --- a/modelscope/msdatasets/dataset_cls/custom_datasets/image_classification/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from typing import TYPE_CHECKING - -from modelscope.utils.import_utils import LazyImportModule - -if TYPE_CHECKING: - from .classification_dataset import ClsDataset - -else: - _import_structure = {'classification_dataset': ['ClsDataset']} - - import sys - - sys.modules[__name__] = LazyImportModule( - __name__, - globals()['__file__'], - _import_structure, - module_spec=__spec__, - extra_objects={}, - ) diff --git a/modelscope/msdatasets/dataset_cls/custom_datasets/image_classification/classification_dataset.py b/modelscope/msdatasets/dataset_cls/custom_datasets/image_classification/classification_dataset.py deleted file mode 100644 index 386810c7..00000000 --- a/modelscope/msdatasets/dataset_cls/custom_datasets/image_classification/classification_dataset.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from easycv.datasets.classification import ClsDataset as _ClsDataset - -from modelscope.metainfo import CustomDatasets -from modelscope.msdatasets.dataset_cls.custom_datasets import CUSTOM_DATASETS -from modelscope.msdatasets.dataset_cls.custom_datasets.easycv_base import \ - EasyCVBaseDataset -from modelscope.utils.constant import Tasks - - -@CUSTOM_DATASETS.register_module( - group_key=Tasks.image_classification, - module_name=CustomDatasets.ClsDataset) -class ClsDataset(_ClsDataset): - """EasyCV dataset for classification. - - Args: - split_config (dict): Dataset root path from MSDataset, e.g. - {"train":"local cache path"} or {"evaluation":"local cache path"}. - preprocessor (Preprocessor): An optional preprocessor instance, please make sure the preprocessor fits for - the model if supplied. Not support yet. - mode: Training or Evaluation. - """ - - def __init__(self, - split_config=None, - preprocessor=None, - mode=None, - *args, - **kwargs) -> None: - EasyCVBaseDataset.__init__( - self, - split_config=split_config, - preprocessor=preprocessor, - mode=mode, - args=args, - kwargs=kwargs) - _ClsDataset.__init__(self, *args, **kwargs) diff --git a/modelscope/msdatasets/dataset_cls/custom_datasets/image_semantic_segmentation/__init__.py b/modelscope/msdatasets/dataset_cls/custom_datasets/image_semantic_segmentation/__init__.py deleted file mode 100644 index 26121bdb..00000000 --- a/modelscope/msdatasets/dataset_cls/custom_datasets/image_semantic_segmentation/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from typing import TYPE_CHECKING - -from modelscope.utils.import_utils import LazyImportModule - -if TYPE_CHECKING: - from .segmentation_dataset import SegDataset - -else: - _import_structure = {'easycv_segmentation': ['SegDataset']} - - import sys - - sys.modules[__name__] = LazyImportModule( - __name__, - globals()['__file__'], - _import_structure, - module_spec=__spec__, - extra_objects={}, - ) diff --git a/modelscope/msdatasets/dataset_cls/custom_datasets/image_semantic_segmentation/segmentation_dataset.py b/modelscope/msdatasets/dataset_cls/custom_datasets/image_semantic_segmentation/segmentation_dataset.py deleted file mode 100644 index 71e7c42b..00000000 --- a/modelscope/msdatasets/dataset_cls/custom_datasets/image_semantic_segmentation/segmentation_dataset.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from easycv.datasets.segmentation import SegDataset as _SegDataset - -from modelscope.metainfo import CustomDatasets -from modelscope.msdatasets.dataset_cls.custom_datasets import CUSTOM_DATASETS -from modelscope.msdatasets.dataset_cls.custom_datasets.easycv_base import \ - EasyCVBaseDataset -from modelscope.utils.constant import Tasks - - -@CUSTOM_DATASETS.register_module( - group_key=Tasks.image_segmentation, module_name=CustomDatasets.SegDataset) -class SegDataset(EasyCVBaseDataset, _SegDataset): - """EasyCV dataset for Sementic segmentation. - For more details, please refer to : - https://github.com/alibaba/EasyCV/blob/master/easycv/datasets/segmentation/raw.py . - - Args: - split_config (dict): Dataset root path from MSDataset, e.g. - {"train":"local cache path"} or {"evaluation":"local cache path"}. - preprocessor (Preprocessor): An optional preprocessor instance, please make sure the preprocessor fits for - the model if supplied. Not support yet. - mode: Training or Evaluation. - data_source: Data source config to parse input data. - pipeline: Sequence of transform object or config dict to be composed. - ignore_index (int): Label index to be ignored. - profiling: If set True, will print transform time. - """ - - def __init__(self, - split_config=None, - preprocessor=None, - mode=None, - *args, - **kwargs) -> None: - EasyCVBaseDataset.__init__( - self, - split_config=split_config, - preprocessor=preprocessor, - mode=mode, - args=args, - kwargs=kwargs) - _SegDataset.__init__(self, *args, **kwargs) diff --git a/modelscope/msdatasets/dataset_cls/custom_datasets/object_detection/__init__.py b/modelscope/msdatasets/dataset_cls/custom_datasets/object_detection/__init__.py deleted file mode 100644 index 403163e9..00000000 --- a/modelscope/msdatasets/dataset_cls/custom_datasets/object_detection/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from typing import TYPE_CHECKING - -from modelscope.utils.import_utils import LazyImportModule - -if TYPE_CHECKING: - from .detection_dataset import DetDataset, DetImagesMixDataset - -else: - _import_structure = { - 'detection_dataset': ['DetDataset', 'DetImagesMixDataset'] - } - - import sys - - sys.modules[__name__] = LazyImportModule( - __name__, - globals()['__file__'], - _import_structure, - module_spec=__spec__, - extra_objects={}, - ) diff --git a/modelscope/msdatasets/dataset_cls/custom_datasets/object_detection/detection_dataset.py b/modelscope/msdatasets/dataset_cls/custom_datasets/object_detection/detection_dataset.py deleted file mode 100644 index 66c11f64..00000000 --- a/modelscope/msdatasets/dataset_cls/custom_datasets/object_detection/detection_dataset.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. - -from easycv.datasets.detection import DetDataset as _DetDataset -from easycv.datasets.detection import \ - DetImagesMixDataset as _DetImagesMixDataset - -from modelscope.metainfo import CustomDatasets -from modelscope.msdatasets.dataset_cls.custom_datasets import CUSTOM_DATASETS -from modelscope.msdatasets.dataset_cls.custom_datasets.easycv_base import \ - EasyCVBaseDataset -from modelscope.utils.constant import Tasks - - -@CUSTOM_DATASETS.register_module( - group_key=Tasks.image_object_detection, - module_name=CustomDatasets.DetDataset) -@CUSTOM_DATASETS.register_module( - group_key=Tasks.image_segmentation, module_name=CustomDatasets.DetDataset) -class DetDataset(EasyCVBaseDataset, _DetDataset): - """EasyCV dataset for object detection. - For more details, please refer to https://github.com/alibaba/EasyCV/blob/master/easycv/datasets/detection/raw.py . - - Args: - split_config (dict): Dataset root path from MSDataset, e.g. - {"train":"local cache path"} or {"evaluation":"local cache path"}. - preprocessor (Preprocessor): An optional preprocessor instance, please make sure the preprocessor fits for - the model if supplied. Not support yet. - mode: Training or Evaluation. - data_source: Data source config to parse input data. - pipeline: Transform config list - profiling: If set True, will print pipeline time - classes: A list of class names, used in evaluation for result and groundtruth visualization - """ - - def __init__(self, - split_config=None, - preprocessor=None, - mode=None, - *args, - **kwargs) -> None: - EasyCVBaseDataset.__init__( - self, - split_config=split_config, - preprocessor=preprocessor, - mode=mode, - args=args, - kwargs=kwargs) - _DetDataset.__init__(self, *args, **kwargs) - - -@CUSTOM_DATASETS.register_module( - group_key=Tasks.image_object_detection, - module_name=CustomDatasets.DetImagesMixDataset) -@CUSTOM_DATASETS.register_module( - group_key=Tasks.domain_specific_object_detection, - module_name=CustomDatasets.DetImagesMixDataset) -class DetImagesMixDataset(EasyCVBaseDataset, _DetImagesMixDataset): - """EasyCV dataset for object detection, a wrapper of multiple images mixed dataset. - Suitable for training on multiple images mixed data augmentation like - mosaic and mixup. For the augmentation pipeline of mixed image data, - the `get_indexes` method needs to be provided to obtain the image - indexes, and you can set `skip_flags` to change the pipeline running - process. At the same time, we provide the `dynamic_scale` parameter - to dynamically change the output image size. - output boxes format: cx, cy, w, h - - For more details, please refer to https://github.com/alibaba/EasyCV/blob/master/easycv/datasets/detection/mix.py . - - Args: - split_config (dict): Dataset root path from MSDataset, e.g. - {"train":"local cache path"} or {"evaluation":"local cache path"}. - preprocessor (Preprocessor): An optional preprocessor instance, please make sure the preprocessor fits for - the model if supplied. Not support yet. - mode: Training or Evaluation. - data_source (:obj:`DetSourceCoco`): Data source config to parse input data. - pipeline (Sequence[dict]): Sequence of transform object or - config dict to be composed. - dynamic_scale (tuple[int], optional): The image scale can be changed - dynamically. Default to None. - skip_type_keys (list[str], optional): Sequence of type string to - be skip pipeline. Default to None. - label_padding: out labeling padding [N, 120, 5] - """ - - def __init__(self, - split_config=None, - preprocessor=None, - mode=None, - *args, - **kwargs) -> None: - EasyCVBaseDataset.__init__( - self, - split_config=split_config, - preprocessor=preprocessor, - mode=mode, - args=args, - kwargs=kwargs) - _DetImagesMixDataset.__init__(self, *args, **kwargs) diff --git a/modelscope/pipelines/cv/__init__.py b/modelscope/pipelines/cv/__init__.py index 54289644..e9d7a785 100644 --- a/modelscope/pipelines/cv/__init__.py +++ b/modelscope/pipelines/cv/__init__.py @@ -9,7 +9,6 @@ if TYPE_CHECKING: from .animal_recognition_pipeline import AnimalRecognitionPipeline from .body_2d_keypoints_pipeline import Body2DKeypointsPipeline from .body_3d_keypoints_pipeline import Body3DKeypointsPipeline - from .hand_2d_keypoints_pipeline import Hand2DKeypointsPipeline from .cmdssl_video_embedding_pipeline import CMDSSLVideoEmbeddingPipeline from .card_detection_pipeline import CardDetectionPipeline from .hicossl_video_embedding_pipeline import HICOSSLVideoEmbeddingPipeline @@ -29,13 +28,10 @@ if TYPE_CHECKING: from .image_classification_pipeline import GeneralImageClassificationPipeline from .image_color_enhance_pipeline import ImageColorEnhancePipeline from .image_colorization_pipeline import ImageColorizationPipeline - from .image_classification_pipeline import ImageClassificationPipeline from .image_denoise_pipeline import ImageDenoisePipeline from .image_deblur_pipeline import ImageDeblurPipeline from .image_instance_segmentation_pipeline import ImageInstanceSegmentationPipeline from .image_matting_pipeline import ImageMattingPipeline - from .image_panoptic_segmentation_pipeline import ImagePanopticSegmentationPipeline - from .image_semantic_segmentation_pipeline import ImagePanopticSegmentationEasyCVPipeline from .image_portrait_enhancement_pipeline import ImagePortraitEnhancementPipeline from .image_reid_person_pipeline import ImageReidPersonPipeline from .image_semantic_segmentation_pipeline import ImageSemanticSegmentationPipeline @@ -46,7 +42,6 @@ if TYPE_CHECKING: from .image_inpainting_pipeline import ImageInpaintingPipeline from .image_paintbyexample_pipeline import ImagePaintbyexamplePipeline from .product_retrieval_embedding_pipeline import ProductRetrievalEmbeddingPipeline - from .realtime_object_detection_pipeline import RealtimeObjectDetectionPipeline from .live_category_pipeline import LiveCategoryPipeline from .ocr_detection_pipeline import OCRDetectionPipeline from .ocr_recognition_pipeline import OCRRecognitionPipeline @@ -59,10 +54,6 @@ if TYPE_CHECKING: from .video_category_pipeline import VideoCategoryPipeline from .virtual_try_on_pipeline import VirtualTryonPipeline from .shop_segmentation_pipleline import ShopSegmentationPipeline - from .easycv_pipelines import (EasyCVDetectionPipeline, - EasyCVSegmentationPipeline, - Face2DKeypointsPipeline, - HumanWholebodyKeypointsPipeline) from .text_driven_segmentation_pipleline import TextDrivenSegmentationPipeline from .movie_scene_segmentation_pipeline import MovieSceneSegmentationPipeline from .mog_face_detection_pipeline import MogFaceDetectionPipeline @@ -123,7 +114,6 @@ else: 'animal_recognition_pipeline': ['AnimalRecognitionPipeline'], 'body_2d_keypoints_pipeline': ['Body2DKeypointsPipeline'], 'body_3d_keypoints_pipeline': ['Body3DKeypointsPipeline'], - 'hand_2d_keypoints_pipeline': ['Hand2DKeypointsPipeline'], 'card_detection_pipeline': ['CardDetectionPipeline'], 'cmdssl_video_embedding_pipeline': ['CMDSSLVideoEmbeddingPipeline'], 'hicossl_video_embedding_pipeline': ['HICOSSLVideoEmbeddingPipeline'], @@ -140,7 +130,7 @@ else: 'face_recognition_onnx_fm_pipeline': ['FaceRecognitionOnnxFmPipeline'], 'general_recognition_pipeline': ['GeneralRecognitionPipeline'], 'image_classification_pipeline': - ['GeneralImageClassificationPipeline', 'ImageClassificationPipeline'], + ['GeneralImageClassificationPipeline'], 'image_cartoon_pipeline': ['ImageCartoonPipeline'], 'image_denoise_pipeline': ['ImageDenoisePipeline'], 'image_deblur_pipeline': ['ImageDeblurPipeline'], @@ -149,10 +139,6 @@ else: 'image_instance_segmentation_pipeline': ['ImageInstanceSegmentationPipeline'], 'image_matting_pipeline': ['ImageMattingPipeline'], - 'image_panoptic_segmentation_pipeline': [ - 'ImagePanopticSegmentationPipeline', - 'ImagePanopticSegmentationEasyCVPipeline' - ], 'image_portrait_enhancement_pipeline': ['ImagePortraitEnhancementPipeline'], 'image_reid_person_pipeline': ['ImageReidPersonPipeline'], @@ -164,8 +150,6 @@ else: ['Image2ImageTranslationPipeline'], 'product_retrieval_embedding_pipeline': ['ProductRetrievalEmbeddingPipeline'], - 'realtime_object_detection_pipeline': - ['RealtimeObjectDetectionPipeline'], 'live_category_pipeline': ['LiveCategoryPipeline'], 'image_to_image_generate_pipeline': ['Image2ImageGenerationPipeline'], 'image_inpainting_pipeline': ['ImageInpaintingPipeline'], @@ -180,12 +164,6 @@ else: 'video_category_pipeline': ['VideoCategoryPipeline'], 'virtual_try_on_pipeline': ['VirtualTryonPipeline'], 'shop_segmentation_pipleline': ['ShopSegmentationPipeline'], - 'easycv_pipelines': [ - 'EasyCVDetectionPipeline', - 'EasyCVSegmentationPipeline', - 'Face2DKeypointsPipeline', - 'HumanWholebodyKeypointsPipeline', - ], 'text_driven_segmentation_pipleline': ['TextDrivenSegmentationPipeline'], 'movie_scene_segmentation_pipeline': @@ -202,9 +180,8 @@ else: ['FaceAttributeRecognitionPipeline'], 'mtcnn_face_detection_pipeline': ['MtcnnFaceDetectionPipeline'], 'hand_static_pipeline': ['HandStaticPipeline'], - 'referring_video_object_segmentation_pipeline': [ - 'ReferringVideoObjectSegmentationPipeline' - ], + 'referring_video_object_segmentation_pipeline': + ['ReferringVideoObjectSegmentationPipeline'], 'language_guided_video_summarization_pipeline': [ 'LanguageGuidedVideoSummarizationPipeline' ], diff --git a/modelscope/pipelines/cv/easycv_pipelines/__init__.py b/modelscope/pipelines/cv/easycv_pipelines/__init__.py deleted file mode 100644 index e0209b85..00000000 --- a/modelscope/pipelines/cv/easycv_pipelines/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from typing import TYPE_CHECKING - -from modelscope.utils.import_utils import LazyImportModule - -if TYPE_CHECKING: - from .detection_pipeline import EasyCVDetectionPipeline - from .segmentation_pipeline import EasyCVSegmentationPipeline - from .face_2d_keypoints_pipeline import Face2DKeypointsPipeline - from .human_wholebody_keypoint_pipeline import HumanWholebodyKeypointsPipeline -else: - _import_structure = { - 'detection_pipeline': ['EasyCVDetectionPipeline'], - 'segmentation_pipeline': ['EasyCVSegmentationPipeline'], - 'face_2d_keypoints_pipeline': ['Face2DKeypointsPipeline'], - 'human_wholebody_keypoint_pipeline': - ['HumanWholebodyKeypointsPipeline'], - } - - import sys - - sys.modules[__name__] = LazyImportModule( - __name__, - globals()['__file__'], - _import_structure, - module_spec=__spec__, - extra_objects={}, - ) diff --git a/modelscope/pipelines/cv/easycv_pipelines/base.py b/modelscope/pipelines/cv/easycv_pipelines/base.py deleted file mode 100644 index 0a31be94..00000000 --- a/modelscope/pipelines/cv/easycv_pipelines/base.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import glob -import os -import os.path as osp -from typing import Any - -import numpy as np -from easycv.utils.ms_utils import EasyCVMeta -from PIL import ImageFile - -from modelscope.hub.snapshot_download import snapshot_download -from modelscope.pipelines.util import is_official_hub_path -from modelscope.utils.config import Config -from modelscope.utils.constant import (DEFAULT_MODEL_REVISION, Invoke, - ModelFile, ThirdParty) -from modelscope.utils.device import create_device - - -class EasyCVPipeline(object): - """Base pipeline for EasyCV. - Loading configuration file of modelscope style by default, - but it is actually use the predictor api of easycv to predict. - So here we do some adaptation work for configuration and predict api. - """ - - def __init__(self, model: str, model_file_pattern='*.pt', *args, **kwargs): - """ - model (str): model id on modelscope hub or local model path. - model_file_pattern (str): model file pattern. - - """ - self.model_file_pattern = model_file_pattern - - assert isinstance(model, str) - if osp.exists(model): - model_dir = model - else: - assert is_official_hub_path( - model), 'Only support local model path and official hub path!' - model_dir = snapshot_download( - model_id=model, - revision=DEFAULT_MODEL_REVISION, - user_agent={ - Invoke.KEY: Invoke.PIPELINE, - ThirdParty.KEY: ThirdParty.EASYCV - }) - - assert osp.isdir(model_dir) - model_files = glob.glob( - os.path.join(model_dir, self.model_file_pattern)) - assert len( - model_files - ) == 1, f'Need one model file, but find {len(model_files)}: {model_files}' - - model_path = model_files[0] - self.model_path = model_path - self.model_dir = model_dir - - # get configuration file from source model dir - self.config_file = os.path.join(model_dir, ModelFile.CONFIGURATION) - assert os.path.exists( - self.config_file - ), f'Not find "{ModelFile.CONFIGURATION}" in model directory!' - - self.cfg = Config.from_file(self.config_file) - if 'device' in kwargs: - kwargs['device'] = create_device(kwargs['device']) - if 'predictor_config' in kwargs: - kwargs.pop('predictor_config') - self.predict_op = self._build_predict_op(**kwargs) - - def _build_predict_op(self, **kwargs): - """Build EasyCV predictor.""" - from easycv.predictors.builder import build_predictor - - easycv_config = self._to_easycv_config() - pipeline_op = build_predictor(self.cfg.pipeline.predictor_config, { - 'model_path': self.model_path, - 'config_file': easycv_config, - **kwargs - }) - return pipeline_op - - def _to_easycv_config(self): - """Adapt to EasyCV predictor.""" - # TODO: refine config compatibility problems - - easycv_arch = self.cfg.model.pop(EasyCVMeta.ARCH, None) - model_cfg = self.cfg.model - # Revert to the configuration of easycv - if easycv_arch is not None: - model_cfg.update(easycv_arch) - - easycv_config = Config(dict(model=model_cfg)) - - reserved_keys = [] - if hasattr(self.cfg, EasyCVMeta.META): - easycv_meta_cfg = getattr(self.cfg, EasyCVMeta.META) - reserved_keys = easycv_meta_cfg.get(EasyCVMeta.RESERVED_KEYS, []) - for key in reserved_keys: - easycv_config.merge_from_dict({key: getattr(self.cfg, key)}) - if 'test_pipeline' not in reserved_keys: - easycv_config.merge_from_dict( - {'test_pipeline': self.cfg.dataset.val.get('pipeline', [])}) - - return easycv_config - - def _is_single_inputs(self, inputs): - if isinstance(inputs, str) or (isinstance(inputs, list) - and len(inputs) == 1) or isinstance( - inputs, np.ndarray) or isinstance( - inputs, ImageFile.ImageFile): - return True - - return False - - def __call__(self, inputs) -> Any: - outputs = self.predict_op(inputs) - - if self._is_single_inputs(inputs): - outputs = outputs[0] - - return outputs diff --git a/modelscope/pipelines/cv/easycv_pipelines/detection_pipeline.py b/modelscope/pipelines/cv/easycv_pipelines/detection_pipeline.py deleted file mode 100644 index 2a95ebb4..00000000 --- a/modelscope/pipelines/cv/easycv_pipelines/detection_pipeline.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from typing import Any - -from modelscope.metainfo import Pipelines -from modelscope.outputs import OutputKeys -from modelscope.pipelines.builder import PIPELINES -from modelscope.utils.constant import ModelFile, Tasks -from modelscope.utils.cv.image_utils import \ - show_image_object_detection_auto_result -from .base import EasyCVPipeline - - -@PIPELINES.register_module( - Tasks.image_object_detection, module_name=Pipelines.easycv_detection) -@PIPELINES.register_module( - Tasks.image_object_detection, - module_name=Pipelines.image_object_detection_auto) -@PIPELINES.register_module( - Tasks.domain_specific_object_detection, - module_name=Pipelines.hand_detection) -class EasyCVDetectionPipeline(EasyCVPipeline): - """Pipeline for easycv detection task.""" - - def __init__(self, - model: str, - model_file_pattern=ModelFile.TORCH_MODEL_FILE, - *args, - **kwargs): - """ - model (str): model id on modelscope hub or local model path. - model_file_pattern (str): model file pattern. - """ - - super(EasyCVDetectionPipeline, self).__init__( - model=model, - model_file_pattern=model_file_pattern, - *args, - **kwargs) - - def show_result(self, img_path, result, save_path=None): - show_image_object_detection_auto_result(img_path, result, save_path) - - def __call__(self, inputs) -> Any: - outputs = self.predict_op(inputs) - - scores = [] - labels = [] - boxes = [] - for output in outputs: - for score, label, box in zip(output['detection_scores'], - output['detection_classes'], - output['detection_boxes']): - scores.append(score) - labels.append(self.cfg.CLASSES[label]) - boxes.append([b for b in box]) - - results = [{ - OutputKeys.SCORES: scores, - OutputKeys.LABELS: labels, - OutputKeys.BOXES: boxes - } for output in outputs] - - if self._is_single_inputs(inputs): - results = results[0] - - return results diff --git a/modelscope/pipelines/cv/easycv_pipelines/face_2d_keypoints_pipeline.py b/modelscope/pipelines/cv/easycv_pipelines/face_2d_keypoints_pipeline.py deleted file mode 100644 index 0ddc6a6c..00000000 --- a/modelscope/pipelines/cv/easycv_pipelines/face_2d_keypoints_pipeline.py +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import copy -import math -from typing import Any - -import cv2 -import numpy as np - -from modelscope.metainfo import Pipelines -from modelscope.outputs import OutputKeys -from modelscope.pipelines import pipeline -from modelscope.pipelines.builder import PIPELINES -from modelscope.preprocessors import LoadImage -from modelscope.utils.constant import ModelFile, Tasks -from modelscope.utils.logger import get_logger -from .base import EasyCVPipeline - -logger = get_logger() - - -@PIPELINES.register_module( - Tasks.face_2d_keypoints, module_name=Pipelines.face_2d_keypoints) -class Face2DKeypointsPipeline(EasyCVPipeline): - """Pipeline for face 2d keypoints detection.""" - - def __init__(self, - model: str, - model_file_pattern=ModelFile.TORCH_MODEL_FILE, - *args, - **kwargs): - """ - model (str): model id on modelscope hub or local model path. - model_file_pattern (str): model file pattern. - """ - - super(Face2DKeypointsPipeline, self).__init__( - model=model, - model_file_pattern=model_file_pattern, - *args, - **kwargs) - - # face detect pipeline - det_model_id = 'damo/cv_resnet_facedetection_scrfd10gkps' - self.face_detection = pipeline( - Tasks.face_detection, model=det_model_id) - - def show_result(self, img, points, scale=2, save_path=None): - return self.predict_op.show_result(img, points, scale, save_path) - - def _choose_face(self, det_result, min_face=10): - """ - choose face with maximum area - Args: - det_result: output of face detection pipeline - min_face: minimum size of valid face w/h - """ - bboxes = np.array(det_result[OutputKeys.BOXES]) - landmarks = np.array(det_result[OutputKeys.KEYPOINTS]) - if bboxes.shape[0] == 0: - logger.warning('No face detected!') - return None - # face idx with enough size - face_idx = [] - for i in range(bboxes.shape[0]): - box = bboxes[i] - if (box[2] - box[0]) >= min_face and (box[3] - box[1]) >= min_face: - face_idx += [i] - if len(face_idx) == 0: - logger.warning( - f'Face size not enough, less than {min_face}x{min_face}!') - return None - bboxes = bboxes[face_idx] - landmarks = landmarks[face_idx] - - return bboxes, landmarks - - def expend_box(self, box, w, h, scalex=0.3, scaley=0.5): - x1 = box[0] - y1 = box[1] - wb = box[2] - x1 - hb = box[3] - y1 - deltax = int(wb * scalex) - deltay1 = int(hb * scaley) - deltay2 = int(hb * scalex) - x1 = x1 - deltax - y1 = y1 - deltay1 - if x1 < 0: - deltax = deltax + x1 - x1 = 0 - if y1 < 0: - deltay1 = deltay1 + y1 - y1 = 0 - x2 = x1 + wb + 2 * deltax - y2 = y1 + hb + deltay1 + deltay2 - x2 = np.clip(x2, 0, w - 1) - y2 = np.clip(y2, 0, h - 1) - return [x1, y1, x2, y2] - - def rotate_point(self, angle, center, landmark): - rad = angle * np.pi / 180.0 - alpha = np.cos(rad) - beta = np.sin(rad) - M = np.zeros((2, 3), dtype=np.float32) - M[0, 0] = alpha - M[0, 1] = beta - M[0, 2] = (1 - alpha) * center[0] - beta * center[1] - M[1, 0] = -beta - M[1, 1] = alpha - M[1, 2] = beta * center[0] + (1 - alpha) * center[1] - - landmark_ = np.asarray([(M[0, 0] * x + M[0, 1] * y + M[0, 2], - M[1, 0] * x + M[1, 1] * y + M[1, 2]) - for (x, y) in landmark]) - return M, landmark_ - - def rotate_crop_img(self, img, pts, M): - imgT = cv2.warpAffine(img, M, (int(img.shape[1]), int(img.shape[0]))) - - x1 = pts[5][0] - x2 = pts[5][0] - y1 = pts[5][1] - y2 = pts[5][1] - for i in range(0, 9): - x1 = min(x1, pts[i][0]) - x2 = max(x2, pts[i][0]) - y1 = min(y1, pts[i][1]) - y2 = max(y2, pts[i][1]) - - height, width, _ = imgT.shape - x1 = min(max(0, int(x1)), width) - y1 = min(max(0, int(y1)), height) - x2 = min(max(0, int(x2)), width) - y2 = min(max(0, int(y2)), height) - sub_imgT = imgT[y1:y2, x1:x2] - - return sub_imgT, imgT, [x1, y1, x2, y2] - - def crop_img(self, imgT, pts): - enlarge_ratio = 1.1 - - x1 = np.min(pts[:, 0]) - x2 = np.max(pts[:, 0]) - y1 = np.min(pts[:, 1]) - y2 = np.max(pts[:, 1]) - w = x2 - x1 + 1 - h = y2 - y1 + 1 - x1 = int(x1 - (enlarge_ratio - 1.0) / 2.0 * w) - y1 = int(y1 - (enlarge_ratio - 1.0) / 2.0 * h) - x1 = max(0, x1) - y1 = max(0, y1) - - new_w = int(enlarge_ratio * w) - new_h = int(enlarge_ratio * h) - new_x1 = x1 - new_y1 = y1 - new_x2 = new_x1 + new_w - new_y2 = new_y1 + new_h - - height, width, _ = imgT.shape - - new_x1 = min(max(0, new_x1), width) - new_y1 = min(max(0, new_y1), height) - new_x2 = max(min(width, new_x2), 0) - new_y2 = max(min(height, new_y2), 0) - - sub_imgT = imgT[new_y1:new_y2, new_x1:new_x2] - - return sub_imgT, [new_x1, new_y1, new_x2, new_y2] - - def __call__(self, inputs) -> Any: - img = LoadImage.convert_to_ndarray(inputs) - h, w, c = img.shape - img_rgb = copy.deepcopy(img) - img_rgb = img_rgb[:, :, ::-1] - det_result = self.face_detection(img_rgb) - - bboxes = np.array(det_result[OutputKeys.BOXES]) - if bboxes.shape[0] == 0: - logger.warning('No face detected!') - results = { - OutputKeys.KEYPOINTS: [], - OutputKeys.POSES: [], - OutputKeys.BOXES: [] - } - return results - - boxes, keypoints = self._choose_face(det_result) - - output_boxes = [] - output_keypoints = [] - output_poses = [] - for index, box_ori in enumerate(boxes): - box = self.expend_box(box_ori, w, h, scalex=0.1, scaley=0.1) - y0 = int(box[1]) - y1 = int(box[3]) - x0 = int(box[0]) - x1 = int(box[2]) - sub_img = img[y0:y1, x0:x1] - - keypoint = keypoints[index] - pts = [[keypoint[0], keypoint[1]], [keypoint[2], keypoint[3]], - [keypoint[4], keypoint[5]], [keypoint[6], keypoint[7]], - [keypoint[8], keypoint[9]], [box[0], box[1]], - [box[2], box[1]], [box[0], box[3]], [box[2], box[3]]] - # radian - angle = math.atan2((pts[1][1] - pts[0][1]), - (pts[1][0] - pts[0][0])) - # angle - theta = angle * (180 / np.pi) - - center = [w // 2, h // 2] - cx, cy = center - M, landmark_ = self.rotate_point(theta, (cx, cy), pts) - sub_imgT, imgT, bbox = self.rotate_crop_img(img, landmark_, M) - - outputs = self.predict_op([sub_imgT])[0] - tmp_keypoints = outputs['point'] - - for idx in range(0, len(tmp_keypoints)): - tmp_keypoints[idx][0] += bbox[0] - tmp_keypoints[idx][1] += bbox[1] - - for idx in range(0, 6): - sub_img, bbox = self.crop_img(imgT, tmp_keypoints) - outputs = self.predict_op([sub_img])[0] - tmp_keypoints = outputs['point'] - for idx in range(0, len(tmp_keypoints)): - tmp_keypoints[idx][0] += bbox[0] - tmp_keypoints[idx][1] += bbox[1] - - M2, tmp_keypoints = self.rotate_point(-theta, (cx, cy), - tmp_keypoints) - - output_keypoints.append(np.array(tmp_keypoints)) - output_poses.append(np.array(outputs['pose'])) - output_boxes.append(np.array(box_ori)) - - results = { - OutputKeys.KEYPOINTS: output_keypoints, - OutputKeys.POSES: output_poses, - OutputKeys.BOXES: output_boxes - } - - return results diff --git a/modelscope/pipelines/cv/easycv_pipelines/human_wholebody_keypoint_pipeline.py b/modelscope/pipelines/cv/easycv_pipelines/human_wholebody_keypoint_pipeline.py deleted file mode 100644 index 903c4106..00000000 --- a/modelscope/pipelines/cv/easycv_pipelines/human_wholebody_keypoint_pipeline.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import os.path -from typing import Any - -from modelscope.metainfo import Pipelines -from modelscope.outputs import OutputKeys -from modelscope.pipelines.builder import PIPELINES -from modelscope.utils.constant import ModelFile, Tasks -from .base import EasyCVPipeline - - -@PIPELINES.register_module( - Tasks.human_wholebody_keypoint, - module_name=Pipelines.human_wholebody_keypoint) -class HumanWholebodyKeypointsPipeline(EasyCVPipeline): - """Pipeline for human wholebody 2d keypoints detection.""" - - def __init__(self, - model: str, - model_file_pattern=ModelFile.TORCH_MODEL_FILE, - *args, - **kwargs): - """ - model (str): model id on modelscope hub or local model path. - model_file_pattern (str): model file pattern. - """ - super(HumanWholebodyKeypointsPipeline, self).__init__( - model=model, - model_file_pattern=model_file_pattern, - *args, - **kwargs) - - def _build_predict_op(self, **kwargs): - """Build EasyCV predictor.""" - from easycv.predictors.builder import build_predictor - detection_predictor_type = self.cfg['DETECTION']['type'] - detection_model_path = os.path.join( - self.model_dir, self.cfg['DETECTION']['model_path']) - detection_cfg_file = os.path.join(self.model_dir, - self.cfg['DETECTION']['config_file']) - detection_score_threshold = self.cfg['DETECTION']['score_threshold'] - self.cfg.pipeline.predictor_config[ - 'detection_predictor_config'] = dict( - type=detection_predictor_type, - model_path=detection_model_path, - config_file=detection_cfg_file, - score_threshold=detection_score_threshold) - easycv_config = self._to_easycv_config() - pipeline_op = build_predictor(self.cfg.pipeline.predictor_config, { - 'model_path': self.model_path, - 'config_file': easycv_config, - **kwargs - }) - return pipeline_op - - def __call__(self, inputs) -> Any: - outputs = self.predict_op(inputs) - - results = [{ - OutputKeys.KEYPOINTS: output['keypoints'], - OutputKeys.BOXES: output['boxes'] - } for output in outputs] - - if self._is_single_inputs(inputs): - results = results[0] - - return results diff --git a/modelscope/pipelines/cv/easycv_pipelines/segmentation_pipeline.py b/modelscope/pipelines/cv/easycv_pipelines/segmentation_pipeline.py deleted file mode 100644 index bd09fc9b..00000000 --- a/modelscope/pipelines/cv/easycv_pipelines/segmentation_pipeline.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from typing import Any - -import numpy as np - -from modelscope.metainfo import Pipelines -from modelscope.outputs import OutputKeys -from modelscope.pipelines.builder import PIPELINES -from modelscope.utils.constant import Tasks -from .base import EasyCVPipeline - - -@PIPELINES.register_module( - Tasks.image_segmentation, module_name=Pipelines.easycv_segmentation) -class EasyCVSegmentationPipeline(EasyCVPipeline): - """Pipeline for easycv segmentation task.""" - - def __init__(self, model: str, model_file_pattern='*.pt', *args, **kwargs): - """ - model (str): model id on modelscope hub or local model path. - model_file_pattern (str): model file pattern. - """ - - super(EasyCVSegmentationPipeline, self).__init__( - model=model, - model_file_pattern=model_file_pattern, - *args, - **kwargs) - - def __call__(self, inputs) -> Any: - outputs = self.predict_op(inputs) - - semantic_result = outputs[0]['seg_pred'] - - ids = np.unique(semantic_result)[::-1] - legal_indices = ids != len(self.predict_op.CLASSES) # for VOID label - ids = ids[legal_indices] - segms = (semantic_result[None] == ids[:, None, None]) - masks = [it.astype(np.int) for it in segms] - labels_txt = np.array(self.predict_op.CLASSES)[ids].tolist() - - results = { - OutputKeys.MASKS: masks, - OutputKeys.LABELS: labels_txt, - OutputKeys.SCORES: [0.999 for _ in range(len(labels_txt))] - } - return results diff --git a/modelscope/pipelines/cv/hand_2d_keypoints_pipeline.py b/modelscope/pipelines/cv/hand_2d_keypoints_pipeline.py deleted file mode 100644 index 63281e80..00000000 --- a/modelscope/pipelines/cv/hand_2d_keypoints_pipeline.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import os.path - -from modelscope.metainfo import Pipelines -from modelscope.pipelines.builder import PIPELINES -from modelscope.utils.constant import ModelFile, Tasks -from .easycv_pipelines.base import EasyCVPipeline - - -@PIPELINES.register_module( - Tasks.hand_2d_keypoints, module_name=Pipelines.hand_2d_keypoints) -class Hand2DKeypointsPipeline(EasyCVPipeline): - """Pipeline for hand pose keypoint task.""" - - def __init__(self, - model: str, - model_file_pattern=ModelFile.TORCH_MODEL_FILE, - *args, - **kwargs): - """ - model (str): model id on modelscope hub or local model path. - model_file_pattern (str): model file pattern. - """ - super(Hand2DKeypointsPipeline, self).__init__( - model=model, - model_file_pattern=model_file_pattern, - *args, - **kwargs) - - def _build_predict_op(self, **kwargs): - """Build EasyCV predictor.""" - from easycv.predictors.builder import build_predictor - detection_predictor_type = self.cfg['DETECTION']['type'] - detection_model_path = os.path.join( - self.model_dir, self.cfg['DETECTION']['model_path']) - detection_cfg_file = os.path.join(self.model_dir, - self.cfg['DETECTION']['config_file']) - detection_score_threshold = self.cfg['DETECTION']['score_threshold'] - self.cfg.pipeline.predictor_config[ - 'detection_predictor_config'] = dict( - type=detection_predictor_type, - model_path=detection_model_path, - config_file=detection_cfg_file, - score_threshold=detection_score_threshold) - easycv_config = self._to_easycv_config() - pipeline_op = build_predictor(self.cfg.pipeline.predictor_config, { - 'model_path': self.model_path, - 'config_file': easycv_config, - **kwargs - }) - return pipeline_op diff --git a/modelscope/pipelines/cv/image_panoptic_segmentation_pipeline.py b/modelscope/pipelines/cv/image_panoptic_segmentation_pipeline.py deleted file mode 100644 index fe941d9f..00000000 --- a/modelscope/pipelines/cv/image_panoptic_segmentation_pipeline.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from typing import Any, Dict, Union - -import cv2 -import numpy as np -import PIL -import torch - -from modelscope.metainfo import Pipelines -from modelscope.outputs import OutputKeys -from modelscope.pipelines.base import Input, Pipeline -from modelscope.pipelines.builder import PIPELINES -from modelscope.pipelines.cv.easycv_pipelines.base import EasyCVPipeline -from modelscope.preprocessors import load_image -from modelscope.utils.constant import Tasks -from modelscope.utils.logger import get_logger - -logger = get_logger() - - -@PIPELINES.register_module( - Tasks.image_segmentation, - module_name=Pipelines.image_panoptic_segmentation) -class ImagePanopticSegmentationPipeline(Pipeline): - - def __init__(self, model: str, **kwargs): - """ - use `model` to create a image panoptic segmentation pipeline for prediction - Args: - model: model id on modelscope hub. - """ - super().__init__(model=model, **kwargs) - - logger.info('panoptic segmentation model, pipeline init') - - def preprocess(self, input: Input) -> Dict[str, Any]: - from mmdet.datasets.pipelines import Compose - from mmcv.parallel import collate, scatter - from mmdet.datasets import replace_ImageToTensor - - cfg = self.model.cfg - # build the data pipeline - - if isinstance(input, str): - cfg.data.test.pipeline[0].type = 'LoadImageFromWebcam' - img = np.array(load_image(input)) - img = img[:, :, ::-1] # convert to bgr - elif isinstance(input, PIL.Image.Image): - cfg.data.test.pipeline[0].type = 'LoadImageFromWebcam' - img = np.array(input.convert('RGB')) - elif isinstance(input, np.ndarray): - cfg.data.test.pipeline[0].type = 'LoadImageFromWebcam' - if len(input.shape) == 2: - img = cv2.cvtColor(input, cv2.COLOR_GRAY2BGR) - else: - img = input - else: - raise TypeError(f'input should be either str, PIL.Image,' - f' np.array, but got {type(input)}') - - # collect data - data = dict(img=img) - cfg.data.test.pipeline = replace_ImageToTensor(cfg.data.test.pipeline) - test_pipeline = Compose(cfg.data.test.pipeline) - - data = test_pipeline(data) - # copy from mmdet_model collect data - data = collate([data], samples_per_gpu=1) - data['img_metas'] = [ - img_metas.data[0] for img_metas in data['img_metas'] - ] - data['img'] = [img.data[0] for img in data['img']] - if next(self.model.parameters()).is_cuda: - # scatter to specified GPU - data = scatter(data, [next(self.model.parameters()).device])[0] - - return data - - def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: - results = self.model.inference(input) - - return results - - def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: - # bz=1, tcguo - pan_results = inputs[0]['pan_results'] - INSTANCE_OFFSET = 1000 - - ids = np.unique(pan_results)[::-1] - legal_indices = ids != self.model.num_classes # for VOID label - ids = ids[legal_indices] - labels = np.array([id % INSTANCE_OFFSET for id in ids], dtype=np.int64) - segms = (pan_results[None] == ids[:, None, None]) - masks = [it.astype(np.int) for it in segms] - labels_txt = np.array(self.model.CLASSES)[labels].tolist() - - outputs = { - OutputKeys.MASKS: masks, - OutputKeys.LABELS: labels_txt, - OutputKeys.SCORES: [0.999 for _ in range(len(labels_txt))] - } - return outputs - - -@PIPELINES.register_module( - Tasks.image_segmentation, - module_name=Pipelines.image_panoptic_segmentation_easycv) -class ImagePanopticSegmentationEasyCVPipeline(EasyCVPipeline): - """Pipeline built upon easycv for image segmentation.""" - - def __init__(self, model: str, model_file_pattern='*.pt', *args, **kwargs): - """ - model (str): model id on modelscope hub or local model path. - model_file_pattern (str): model file pattern. - """ - super(ImagePanopticSegmentationEasyCVPipeline, self).__init__( - model=model, - model_file_pattern=model_file_pattern, - *args, - **kwargs) - - def __call__(self, inputs) -> Any: - outputs = self.predict_op(inputs) - easycv_results = outputs[0] - - results = { - OutputKeys.MASKS: - easycv_results[OutputKeys.MASKS], - OutputKeys.LABELS: - easycv_results[OutputKeys.LABELS], - OutputKeys.SCORES: - [0.999 for _ in range(len(easycv_results[OutputKeys.LABELS]))] - } - - return results diff --git a/modelscope/trainers/easycv/__init__.py b/modelscope/trainers/easycv/__init__.py deleted file mode 100644 index b1b8fc15..00000000 --- a/modelscope/trainers/easycv/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from typing import TYPE_CHECKING - -from modelscope.utils.import_utils import LazyImportModule - -if TYPE_CHECKING: - from .utils import AddLrLogHook, EasyCVMetric -else: - _import_structure = {'utils': ['AddLrLogHook', 'EasyCVMetric']} - - import sys - - sys.modules[__name__] = LazyImportModule( - __name__, - globals()['__file__'], - _import_structure, - module_spec=__spec__, - extra_objects={}, - ) diff --git a/modelscope/trainers/easycv/trainer.py b/modelscope/trainers/easycv/trainer.py deleted file mode 100644 index 58d6a440..00000000 --- a/modelscope/trainers/easycv/trainer.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from copy import deepcopy -from functools import partial -from typing import Callable, Optional, Tuple, Union - -import torch -from easycv.utils.checkpoint import load_checkpoint as ev_load_checkpoint -from torch import nn -from torch.utils.data import Dataset - -from modelscope.metainfo import Trainers -from modelscope.models.base import TorchModel -from modelscope.msdatasets import MsDataset -from modelscope.preprocessors import Preprocessor -from modelscope.trainers import EpochBasedTrainer -from modelscope.trainers.base import TRAINERS -from modelscope.trainers.easycv.utils import register_util -from modelscope.trainers.hooks import HOOKS -from modelscope.trainers.parallel.builder import build_parallel -from modelscope.trainers.parallel.utils import is_parallel -from modelscope.utils.config import Config -from modelscope.utils.constant import DEFAULT_MODEL_REVISION -from modelscope.utils.import_utils import LazyImportModule -from modelscope.utils.registry import default_group - - -@TRAINERS.register_module(module_name=Trainers.easycv) -class EasyCVEpochBasedTrainer(EpochBasedTrainer): - """Epoch based Trainer for EasyCV. - - Args: - cfg_file(str): The config file of EasyCV. - model (:obj:`torch.nn.Module` or :obj:`TorchModel` or `str`): The model to be run, or a valid model dir - or a model id. If model is None, build_model method will be called. - train_dataset (`MsDataset` or `torch.utils.data.Dataset`, *optional*): - The dataset to use for training. - Note that if it's a `torch.utils.data.IterableDataset` with some randomization and you are training in a - distributed fashion, your iterable dataset should either use a internal attribute `generator` that is a - `torch.Generator` for the randomization that must be identical on all processes (and the Trainer will - manually set the seed of this `generator` at each epoch) or have a `set_epoch()` method that internally - sets the seed of the RNGs used. - eval_dataset (`MsDataset` or `torch.utils.data.Dataset`, *optional*): The dataset to use for evaluation. - preprocessor (:obj:`Preprocessor`, *optional*): The optional preprocessor. - NOTE: If the preprocessor has been called before the dataset fed into this trainer by user's custom code, - this parameter should be None, meanwhile remove the 'preprocessor' key from the cfg_file. - Else the preprocessor will be instantiated from the cfg_file or assigned from this parameter and - this preprocessing action will be executed every time the dataset's __getitem__ is called. - optimizers (`Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler._LRScheduler]`, *optional*): A tuple - containing the optimizer and the scheduler to use. - max_epochs: (int, optional): Total training epochs. - """ - - def __init__( - self, - cfg_file: Optional[str] = None, - model: Optional[Union[TorchModel, nn.Module, str]] = None, - arg_parse_fn: Optional[Callable] = None, - train_dataset: Optional[Union[MsDataset, Dataset]] = None, - eval_dataset: Optional[Union[MsDataset, Dataset]] = None, - preprocessor: Optional[Preprocessor] = None, - optimizers: Tuple[torch.optim.Optimizer, - torch.optim.lr_scheduler._LRScheduler] = (None, - None), - model_revision: Optional[str] = DEFAULT_MODEL_REVISION, - **kwargs): - - register_util.register_parallel() - register_util.register_part_mmcv_hooks_to_ms() - - super(EasyCVEpochBasedTrainer, self).__init__( - model=model, - cfg_file=cfg_file, - arg_parse_fn=arg_parse_fn, - preprocessor=preprocessor, - optimizers=optimizers, - model_revision=model_revision, - train_dataset=train_dataset, - eval_dataset=eval_dataset, - **kwargs) - - # reset data_collator - from mmcv.parallel import collate - - self.train_data_collator = partial( - collate, - samples_per_gpu=self.cfg.train.dataloader.batch_size_per_gpu) - self.eval_data_collator = partial( - collate, - samples_per_gpu=self.cfg.evaluation.dataloader.batch_size_per_gpu) - - # load pretrained model - load_from = self.cfg.get('load_from', None) - if load_from is not None: - ev_load_checkpoint( - self.model, - filename=load_from, - map_location=self.device, - strict=False, - ) - - # reset parallel - if not self._dist: - assert not is_parallel( - self.model - ), 'Not support model wrapped by custom parallel if not in distributed mode!' - dp_cfg = dict( - type='MMDataParallel', - module=self.model, - device_ids=[torch.cuda.current_device()]) - self.model = build_parallel(dp_cfg) - - def rebuild_config(self, cfg: Config): - cfg = super().rebuild_config(cfg) - # Register easycv hooks dynamicly. If the hook already exists in modelscope, - # the hook in modelscope will be used, otherwise register easycv hook into ms. - # We must manually trigger lazy import to detect whether the hook is in modelscope. - # TODO: use ast index to detect whether the hook is in modelscope - for h_i in cfg.train.get('hooks', []): - sig = ('HOOKS', default_group, h_i['type']) - LazyImportModule.import_module(sig) - if h_i['type'] not in HOOKS._modules[default_group]: - if h_i['type'] in [ - 'TensorboardLoggerHookV2', 'WandbLoggerHookV2' - ]: - raise ValueError( - 'Not support hook %s now, we will support it in the future!' - % h_i['type']) - register_util.register_hook_to_ms(h_i['type']) - return cfg - - def create_optimizer_and_scheduler(self): - """ Create optimizer and lr scheduler - """ - optimizer, lr_scheduler = self.optimizers - if optimizer is None: - optimizer_cfg = self.cfg.train.get('optimizer', None) - else: - optimizer_cfg = None - - optim_options = {} - if optimizer_cfg is not None: - optim_options = optimizer_cfg.pop('options', {}) - from easycv.apis.train import build_optimizer - optimizer = build_optimizer(self.model, optimizer_cfg) - - if lr_scheduler is None: - lr_scheduler_cfg = self.cfg.train.get('lr_scheduler', None) - else: - lr_scheduler_cfg = None - - lr_options = {} - # Adapt to mmcv lr scheduler hook. - # Please refer to: https://github.com/open-mmlab/mmcv/blob/master/mmcv/runner/hooks/lr_updater.py - if lr_scheduler_cfg is not None: - assert optimizer is not None - lr_options = lr_scheduler_cfg.pop('options', {}) - assert 'policy' in lr_scheduler_cfg - policy_type = lr_scheduler_cfg.pop('policy') - if policy_type == policy_type.lower(): - policy_type = policy_type.title() - hook_type = policy_type + 'LrUpdaterHook' - lr_scheduler_cfg['type'] = hook_type - - self.cfg.train.lr_scheduler_hook = lr_scheduler_cfg - - self.optimizer = optimizer - self.lr_scheduler = lr_scheduler - - return self.optimizer, self.lr_scheduler, optim_options, lr_options - - def to_parallel(self, model) -> Union[nn.Module, TorchModel]: - if self.cfg.get('parallel', None) is not None: - dp_cfg = deepcopy(self.cfg['parallel']) - dp_cfg.update( - dict(module=model, device_ids=[torch.cuda.current_device()])) - return build_parallel(dp_cfg) - - dp_cfg = dict( - type='MMDistributedDataParallel', - module=model, - device_ids=[torch.cuda.current_device()]) - - return build_parallel(dp_cfg) diff --git a/modelscope/trainers/easycv/utils/__init__.py b/modelscope/trainers/easycv/utils/__init__.py deleted file mode 100644 index 23cfa36a..00000000 --- a/modelscope/trainers/easycv/utils/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from typing import TYPE_CHECKING - -from modelscope.utils.import_utils import LazyImportModule - -if TYPE_CHECKING: - from .hooks import AddLrLogHook - from .metric import EasyCVMetric - -else: - _import_structure = {'hooks': ['AddLrLogHook'], 'metric': ['EasyCVMetric']} - - import sys - - sys.modules[__name__] = LazyImportModule( - __name__, - globals()['__file__'], - _import_structure, - module_spec=__spec__, - extra_objects={}, - ) diff --git a/modelscope/trainers/easycv/utils/hooks.py b/modelscope/trainers/easycv/utils/hooks.py deleted file mode 100644 index 1f1a5c95..00000000 --- a/modelscope/trainers/easycv/utils/hooks.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -from modelscope.trainers.hooks import HOOKS, Priority -from modelscope.trainers.hooks.lr_scheduler_hook import LrSchedulerHook -from modelscope.utils.constant import LogKeys - - -@HOOKS.register_module(module_name='AddLrLogHook') -class AddLrLogHook(LrSchedulerHook): - """For EasyCV to adapt to ModelScope, the lr log of EasyCV is added in the trainer, - but the trainer of ModelScope does not and it is added in the lr scheduler hook. - But The lr scheduler hook used by EasyCV is the hook of mmcv, and there is no lr log. - It will be deleted in the future. - """ - PRIORITY = Priority.NORMAL - - def __init__(self): - pass - - def before_run(self, trainer): - pass - - def after_train_iter(self, trainer): - trainer.log_buffer.output[LogKeys.LR] = self._get_log_lr(trainer) - - def before_train_epoch(self, trainer): - trainer.log_buffer.output[LogKeys.LR] = self._get_log_lr(trainer) - - def after_train_epoch(self, trainer): - pass diff --git a/modelscope/trainers/easycv/utils/metric.py b/modelscope/trainers/easycv/utils/metric.py deleted file mode 100644 index d952ec3e..00000000 --- a/modelscope/trainers/easycv/utils/metric.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import itertools -from typing import Dict - -import numpy as np -import torch - -from modelscope.metrics.base import Metric -from modelscope.metrics.builder import METRICS - - -@METRICS.register_module(module_name='EasyCVMetric') -class EasyCVMetric(Metric): - """Adapt to ModelScope Metric for EasyCV evaluator. - """ - - def __init__(self, trainer=None, evaluators=None, *args, **kwargs): - from easycv.core.evaluation.builder import build_evaluator - - self.trainer = trainer - self.evaluators = build_evaluator(evaluators) - self.preds = [] - self.grountruths = [] - - def add(self, outputs: Dict, inputs: Dict): - self.preds.append(outputs) - del inputs - - def evaluate(self): - results = {} - for _, batch in enumerate(self.preds): - for k, v in batch.items(): - if k not in results: - results[k] = [] - results[k].append(v) - - for k, v in results.items(): - if len(v) == 0: - raise ValueError(f'empty result for {k}') - - if isinstance(v[0], torch.Tensor): - results[k] = torch.cat(v, 0) - elif isinstance(v[0], (list, np.ndarray)): - results[k] = list(itertools.chain.from_iterable(v)) - else: - raise ValueError( - f'value of batch prediction dict should only be tensor or list, {k} type is {v[0]}' - ) - - metric_values = self.trainer.eval_dataset.evaluate( - results, self.evaluators) - return metric_values - - def merge(self, other: 'EasyCVMetric'): - self.preds.extend(other.preds) - - def __getstate__(self): - return self.preds - - def __setstate__(self, state): - self.__init__() - self.preds = state diff --git a/modelscope/trainers/easycv/utils/register_util.py b/modelscope/trainers/easycv/utils/register_util.py deleted file mode 100644 index 04bf719b..00000000 --- a/modelscope/trainers/easycv/utils/register_util.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import inspect -import logging - -from modelscope.trainers.hooks import HOOKS -from modelscope.trainers.parallel.builder import PARALLEL -from modelscope.utils.registry import default_group - - -class _RegisterManager: - - def __init__(self): - self.registries = {} - - def add(self, module, name, group_key=default_group): - if module.name not in self.registries: - self.registries[module.name] = {} - if group_key not in self.registries[module.name]: - self.registries[module.name][group_key] = [] - - self.registries[module.name][group_key].append(name) - - def exists(self, module, name, group_key=default_group): - if self.registries.get(module.name, None) is None: - return False - if self.registries[module.name].get(group_key, None) is None: - return False - if name in self.registries[module.name][group_key]: - return True - - return False - - -_dynamic_register = _RegisterManager() - - -def register_parallel(): - from mmcv.parallel import MMDistributedDataParallel, MMDataParallel - - mmddp = 'MMDistributedDataParallel' - mmdp = 'MMDataParallel' - - if not _dynamic_register.exists(PARALLEL, mmddp): - _dynamic_register.add(PARALLEL, mmddp) - PARALLEL.register_module( - module_name=mmddp, module_cls=MMDistributedDataParallel) - if not _dynamic_register.exists(PARALLEL, mmdp): - _dynamic_register.add(PARALLEL, mmdp) - PARALLEL.register_module(module_name=mmdp, module_cls=MMDataParallel) - - -def register_hook_to_ms(hook_name, logger=None): - """Register EasyCV hook to ModelScope.""" - from easycv.hooks import HOOKS as _EV_HOOKS - - if hook_name not in _EV_HOOKS._module_dict: - raise ValueError( - f'Not found hook "{hook_name}" in EasyCV hook registries!') - - if _dynamic_register.exists(HOOKS, hook_name): - return - _dynamic_register.add(HOOKS, hook_name) - - obj = _EV_HOOKS._module_dict[hook_name] - HOOKS.register_module(module_name=hook_name, module_cls=obj) - - log_str = f'Register hook "{hook_name}" to modelscope hooks.' - logger.info(log_str) if logger is not None else logging.info(log_str) - - -def register_part_mmcv_hooks_to_ms(): - """Register required mmcv hooks to ModelScope. - Currently we only registered all lr scheduler hooks in EasyCV and mmcv. - Please refer to: - EasyCV: https://github.com/alibaba/EasyCV/blob/master/easycv/hooks/lr_update_hook.py - mmcv: https://github.com/open-mmlab/mmcv/blob/master/mmcv/runner/hooks/lr_updater.py - """ - from mmcv.runner.hooks import lr_updater - from mmcv.runner.hooks import HOOKS as _MMCV_HOOKS - from easycv.hooks import StepFixCosineAnnealingLrUpdaterHook, YOLOXLrUpdaterHook - - mmcv_hooks_in_easycv = [('StepFixCosineAnnealingLrUpdaterHook', - StepFixCosineAnnealingLrUpdaterHook), - ('YOLOXLrUpdaterHook', YOLOXLrUpdaterHook)] - - members = inspect.getmembers(lr_updater) - members.extend(mmcv_hooks_in_easycv) - - for name, obj in members: - if name in _MMCV_HOOKS._module_dict: - if _dynamic_register.exists(HOOKS, name): - continue - _dynamic_register.add(HOOKS, name) - HOOKS.register_module( - module_name=name, - module_cls=obj, - ) diff --git a/tests/msdatasets/test_ms_dataset.py b/tests/msdatasets/test_ms_dataset.py index 8ded9a46..52b015ec 100644 --- a/tests/msdatasets/test_ms_dataset.py +++ b/tests/msdatasets/test_ms_dataset.py @@ -195,17 +195,6 @@ class MsDatasetTest(unittest.TestCase): ) print(next(iter(tf_dataset))) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_streaming_load_coco(self): - small_coco_for_test = MsDataset.load( - dataset_name='EasyCV/small_coco_for_test', - split='train', - use_streaming=True, - download_mode=DownloadMode.FORCE_REDOWNLOAD) - dataset_sample_dict = next(iter(small_coco_for_test)) - print(dataset_sample_dict) - assert dataset_sample_dict.values() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_streaming_load_uni_fold(self): """Test case for loading large scale datasets.""" diff --git a/tests/pipelines/easycv_pipelines/__init__.py b/tests/pipelines/easycv_pipelines/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/pipelines/easycv_pipelines/test_panoptic_segmentation_pipeline.py b/tests/pipelines/easycv_pipelines/test_panoptic_segmentation_pipeline.py deleted file mode 100644 index 49e01251..00000000 --- a/tests/pipelines/easycv_pipelines/test_panoptic_segmentation_pipeline.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import unittest - -import cv2 - -from modelscope.outputs import OutputKeys -from modelscope.pipelines import pipeline -from modelscope.utils.constant import Tasks -from modelscope.utils.cv.image_utils import panoptic_seg_masks_to_image -from modelscope.utils.demo_utils import DemoCompatibilityCheck -from modelscope.utils.test_utils import test_level - - -class EasyCVPanopticSegmentationPipelineTest(unittest.TestCase, - DemoCompatibilityCheck): - img_path = 'data/test/images/image_semantic_segmentation.jpg' - - def setUp(self) -> None: - self.task = Tasks.image_segmentation - self.model_id = 'damo/cv_r50_panoptic-segmentation_cocopan' - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_r50(self): - segmentor = pipeline(task=self.task, model=self.model_id) - outputs = segmentor(self.img_path) - draw_img = panoptic_seg_masks_to_image(outputs[OutputKeys.MASKS]) - cv2.imwrite('result.jpg', draw_img) - print('print ' + self.model_id + ' success') - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/pipelines/easycv_pipelines/test_segmentation_pipeline.py b/tests/pipelines/easycv_pipelines/test_segmentation_pipeline.py deleted file mode 100644 index 5f6dac4b..00000000 --- a/tests/pipelines/easycv_pipelines/test_segmentation_pipeline.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import unittest -from distutils.version import LooseVersion - -import cv2 -import easycv -import numpy as np -from PIL import Image - -from modelscope.outputs import OutputKeys -from modelscope.pipelines import pipeline -from modelscope.utils.constant import Tasks -from modelscope.utils.cv.image_utils import semantic_seg_masks_to_image -from modelscope.utils.demo_utils import DemoCompatibilityCheck -from modelscope.utils.test_utils import test_level - - -class EasyCVSegmentationPipelineTest(unittest.TestCase, - DemoCompatibilityCheck): - img_path = 'data/test/images/image_segmentation.jpg' - - def setUp(self) -> None: - self.task = Tasks.image_segmentation - self.model_id = 'damo/cv_segformer-b0_image_semantic-segmentation_coco-stuff164k' - - def _internal_test_(self, model_id): - semantic_seg = pipeline(task=Tasks.image_segmentation, model=model_id) - outputs = semantic_seg(self.img_path) - - draw_img = semantic_seg_masks_to_image(outputs[OutputKeys.MASKS]) - cv2.imwrite('result.jpg', draw_img) - print('test ' + model_id + ' DONE') - - def _internal_test_batch_(self, model_id, num_samples=2, batch_size=2): - # TODO: support in the future - img = np.asarray(Image.open(self.img_path)) - num_samples = num_samples - batch_size = batch_size - semantic_seg = pipeline( - task=Tasks.image_segmentation, - model=model_id, - batch_size=batch_size) - outputs = semantic_seg([self.img_path] * num_samples) - - self.assertEqual(semantic_seg.predict_op.batch_size, batch_size) - self.assertEqual(len(outputs), num_samples) - - for output in outputs: - self.assertListEqual( - list(img.shape)[:2], list(output['seg_pred'].shape)) - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_segformer_b0(self): - model_id = 'damo/cv_segformer-b0_image_semantic-segmentation_coco-stuff164k' - self._internal_test_(model_id) - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_segformer_b1(self): - model_id = 'damo/cv_segformer-b1_image_semantic-segmentation_coco-stuff164k' - self._internal_test_(model_id) - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_segformer_b2(self): - model_id = 'damo/cv_segformer-b2_image_semantic-segmentation_coco-stuff164k' - self._internal_test_(model_id) - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_segformer_b3(self): - model_id = 'damo/cv_segformer-b3_image_semantic-segmentation_coco-stuff164k' - self._internal_test_(model_id) - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_segformer_b4(self): - model_id = 'damo/cv_segformer-b4_image_semantic-segmentation_coco-stuff164k' - self._internal_test_(model_id) - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_segformer_b5(self): - model_id = 'damo/cv_segformer-b5_image_semantic-segmentation_coco-stuff164k' - self._internal_test_(model_id) - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/trainers/easycv/__init__.py b/tests/trainers/easycv/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/trainers/easycv/test_easycv_trainer.py b/tests/trainers/easycv/test_easycv_trainer.py deleted file mode 100644 index 11f9a739..00000000 --- a/tests/trainers/easycv/test_easycv_trainer.py +++ /dev/null @@ -1,238 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import glob -import os -import shutil -import tempfile -import unittest - -import json -import torch - -from modelscope.metainfo import Models, Pipelines, Trainers -from modelscope.msdatasets import MsDataset -from modelscope.trainers import build_trainer -from modelscope.utils.config import Config -from modelscope.utils.constant import LogKeys, ModeKeys, Tasks -from modelscope.utils.logger import get_logger -from modelscope.utils.test_utils import DistributedTestCase, test_level -from modelscope.utils.torch_utils import is_master - - -def train_func(work_dir, dist=False, log_interval=3, imgs_per_gpu=4): - import easycv - config_path = os.path.join( - os.path.dirname(easycv.__file__), - 'configs/detection/yolox/yolox_s_8xb16_300e_coco.py') - - cfg = Config.from_file(config_path) - - cfg.log_config.update( - dict(hooks=[ - dict(type='TextLoggerHook'), - dict(type='TensorboardLoggerHook') - ])) # not support TensorboardLoggerHookV2 - - ms_cfg_file = os.path.join(work_dir, 'ms_yolox_s_8xb16_300e_coco.json') - from easycv.utils.ms_utils import to_ms_config - - if is_master(): - to_ms_config( - cfg, - dump=True, - task=Tasks.image_object_detection, - ms_model_name=Models.yolox, - pipeline_name=Pipelines.easycv_detection, - save_path=ms_cfg_file) - - trainer_name = Trainers.easycv - train_dataset = MsDataset.load( - dataset_name='small_coco_for_test', namespace='EasyCV', split='train') - eval_dataset = MsDataset.load( - dataset_name='small_coco_for_test', - namespace='EasyCV', - split='validation') - - cfg_options = { - 'train.max_epochs': - 2, - 'train.dataloader.batch_size_per_gpu': - imgs_per_gpu, - 'evaluation.dataloader.batch_size_per_gpu': - 2, - 'train.hooks': [ - { - 'type': 'CheckpointHook', - 'interval': 1 - }, - { - 'type': 'EvaluationHook', - 'interval': 1 - }, - { - 'type': 'TextLoggerHook', - 'ignore_rounding_keys': None, - 'interval': log_interval - }, - ] - } - kwargs = dict( - cfg_file=ms_cfg_file, - train_dataset=train_dataset, - eval_dataset=eval_dataset, - work_dir=work_dir, - cfg_options=cfg_options, - launcher='pytorch' if dist else None) - - trainer = build_trainer(trainer_name, kwargs) - trainer.train() - - -@unittest.skipIf(not torch.cuda.is_available(), 'cuda unittest') -class EasyCVTrainerTestSingleGpu(unittest.TestCase): - - def setUp(self): - self.logger = get_logger() - self.logger.info(('Testing %s.%s' % - (type(self).__name__, self._testMethodName))) - self.tmp_dir = tempfile.TemporaryDirectory().name - if not os.path.exists(self.tmp_dir): - os.makedirs(self.tmp_dir) - - def tearDown(self): - super().tearDown() - shutil.rmtree(self.tmp_dir, ignore_errors=True) - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_single_gpu(self): - train_func(self.tmp_dir) - - results_files = os.listdir(self.tmp_dir) - json_files = glob.glob(os.path.join(self.tmp_dir, '*.log.json')) - self.assertEqual(len(json_files), 1) - - with open(json_files[0], 'r', encoding='utf-8') as f: - lines = [i.strip() for i in f.readlines()] - - self.assertDictContainsSubset( - { - LogKeys.MODE: ModeKeys.TRAIN, - LogKeys.EPOCH: 1, - LogKeys.ITER: 3, - LogKeys.LR: 0.00029 - }, json.loads(lines[0])) - self.assertDictContainsSubset( - { - LogKeys.MODE: ModeKeys.EVAL, - LogKeys.EPOCH: 1, - LogKeys.ITER: 10 - }, json.loads(lines[1])) - self.assertDictContainsSubset( - { - LogKeys.MODE: ModeKeys.TRAIN, - LogKeys.EPOCH: 2, - LogKeys.ITER: 3, - LogKeys.LR: 0.00205 - }, json.loads(lines[2])) - self.assertDictContainsSubset( - { - LogKeys.MODE: ModeKeys.EVAL, - LogKeys.EPOCH: 2, - LogKeys.ITER: 10 - }, json.loads(lines[3])) - self.assertIn(f'{LogKeys.EPOCH}_1.pth', results_files) - self.assertIn(f'{LogKeys.EPOCH}_2.pth', results_files) - for i in [0, 2]: - self.assertIn(LogKeys.DATA_LOAD_TIME, lines[i]) - self.assertIn(LogKeys.ITER_TIME, lines[i]) - self.assertIn(LogKeys.MEMORY, lines[i]) - self.assertIn('total_loss', lines[i]) - for i in [1, 3]: - self.assertIn( - 'CocoDetectionEvaluator_DetectionBoxes_Precision/mAP', - lines[i]) - self.assertIn('DetectionBoxes_Precision/mAP', lines[i]) - self.assertIn('DetectionBoxes_Precision/mAP@.50IOU', lines[i]) - self.assertIn('DetectionBoxes_Precision/mAP@.75IOU', lines[i]) - self.assertIn('DetectionBoxes_Precision/mAP (small)', lines[i]) - - -@unittest.skipIf(not torch.cuda.is_available() - or torch.cuda.device_count() <= 1, 'distributed unittest') -class EasyCVTrainerTestMultiGpus(DistributedTestCase): - - def setUp(self): - self.logger = get_logger() - self.logger.info(('Testing %s.%s' % - (type(self).__name__, self._testMethodName))) - self.tmp_dir = tempfile.TemporaryDirectory().name - if not os.path.exists(self.tmp_dir): - os.makedirs(self.tmp_dir) - - def tearDown(self): - super().tearDown() - shutil.rmtree(self.tmp_dir, ignore_errors=True) - - @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') - def test_multi_gpus(self): - self.start( - train_func, - num_gpus=2, - work_dir=self.tmp_dir, - dist=True, - log_interval=2, - imgs_per_gpu=5) - - results_files = os.listdir(self.tmp_dir) - json_files = glob.glob(os.path.join(self.tmp_dir, '*.log.json')) - self.assertEqual(len(json_files), 1) - - with open(json_files[0], 'r', encoding='utf-8') as f: - lines = [i.strip() for i in f.readlines()] - - self.assertDictContainsSubset( - { - LogKeys.MODE: ModeKeys.TRAIN, - LogKeys.EPOCH: 1, - LogKeys.ITER: 2, - LogKeys.LR: 0.0002 - }, json.loads(lines[0])) - self.assertDictContainsSubset( - { - LogKeys.MODE: ModeKeys.EVAL, - LogKeys.EPOCH: 1, - LogKeys.ITER: 5 - }, json.loads(lines[1])) - self.assertDictContainsSubset( - { - LogKeys.MODE: ModeKeys.TRAIN, - LogKeys.EPOCH: 2, - LogKeys.ITER: 2, - LogKeys.LR: 0.0018 - }, json.loads(lines[2])) - self.assertDictContainsSubset( - { - LogKeys.MODE: ModeKeys.EVAL, - LogKeys.EPOCH: 2, - LogKeys.ITER: 5 - }, json.loads(lines[3])) - - self.assertIn(f'{LogKeys.EPOCH}_1.pth', results_files) - self.assertIn(f'{LogKeys.EPOCH}_2.pth', results_files) - - for i in [0, 2]: - self.assertIn(LogKeys.DATA_LOAD_TIME, lines[i]) - self.assertIn(LogKeys.ITER_TIME, lines[i]) - self.assertIn(LogKeys.MEMORY, lines[i]) - self.assertIn('total_loss', lines[i]) - for i in [1, 3]: - self.assertIn( - 'CocoDetectionEvaluator_DetectionBoxes_Precision/mAP', - lines[i]) - self.assertIn('DetectionBoxes_Precision/mAP', lines[i]) - self.assertIn('DetectionBoxes_Precision/mAP@.50IOU', lines[i]) - self.assertIn('DetectionBoxes_Precision/mAP@.75IOU', lines[i]) - self.assertIn('DetectionBoxes_Precision/mAP (small)', lines[i]) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/trainers/easycv/test_easycv_trainer_detection_dino.py b/tests/trainers/easycv/test_easycv_trainer_detection_dino.py deleted file mode 100644 index 90d1f691..00000000 --- a/tests/trainers/easycv/test_easycv_trainer_detection_dino.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import glob -import os -import shutil -import tempfile -import unittest - -import torch - -from modelscope.metainfo import Trainers -from modelscope.msdatasets import MsDataset -from modelscope.trainers import build_trainer -from modelscope.utils.constant import LogKeys -from modelscope.utils.logger import get_logger -from modelscope.utils.test_utils import test_level - - -@unittest.skipIf(not torch.cuda.is_available(), 'cuda unittest') -class EasyCVTrainerTestDetectionDino(unittest.TestCase): - model_id = 'damo/cv_swinl_image-object-detection_dino' - - def setUp(self): - self.logger = get_logger() - self.logger.info(('Testing %s.%s' % - (type(self).__name__, self._testMethodName))) - - def _train(self, tmp_dir): - cfg_options = {'train.max_epochs': 1} - - trainer_name = Trainers.easycv - - train_dataset = MsDataset.load( - dataset_name='small_coco_for_test', - namespace='EasyCV', - split='train') - eval_dataset = MsDataset.load( - dataset_name='small_coco_for_test', - namespace='EasyCV', - split='validation') - - kwargs = dict( - model=self.model_id, - train_dataset=train_dataset, - eval_dataset=eval_dataset, - work_dir=tmp_dir, - cfg_options=cfg_options) - - trainer = build_trainer(trainer_name, kwargs) - trainer.train() - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_trainer_single_gpu(self): - temp_file_dir = tempfile.TemporaryDirectory() - tmp_dir = temp_file_dir.name - if not os.path.exists(tmp_dir): - os.makedirs(tmp_dir) - - self._train(tmp_dir) - - results_files = os.listdir(tmp_dir) - json_files = glob.glob(os.path.join(tmp_dir, '*.log.json')) - self.assertEqual(len(json_files), 1) - self.assertIn(f'{LogKeys.EPOCH}_1.pth', results_files) - - temp_file_dir.cleanup() - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/trainers/easycv/test_easycv_trainer_face_2d_keypoints.py b/tests/trainers/easycv/test_easycv_trainer_face_2d_keypoints.py deleted file mode 100644 index e4f0c57e..00000000 --- a/tests/trainers/easycv/test_easycv_trainer_face_2d_keypoints.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import glob -import os -import shutil -import tempfile -import unittest - -import torch - -from modelscope.metainfo import Trainers -from modelscope.msdatasets import MsDataset -from modelscope.trainers import build_trainer -from modelscope.utils.constant import DownloadMode, LogKeys, Tasks -from modelscope.utils.logger import get_logger -from modelscope.utils.test_utils import test_level - - -@unittest.skipIf(not torch.cuda.is_available(), 'cuda unittest') -class EasyCVTrainerTestFace2DKeypoints(unittest.TestCase): - model_id = 'damo/cv_mobilenet_face-2d-keypoints_alignment' - - def setUp(self): - self.logger = get_logger() - self.logger.info(('Testing %s.%s' % - (type(self).__name__, self._testMethodName))) - - def _train(self, tmp_dir): - cfg_options = {'train.max_epochs': 2} - - trainer_name = Trainers.easycv - - train_dataset = MsDataset.load( - dataset_name='face_2d_keypoints_dataset', - namespace='modelscope', - split='train', - download_mode=DownloadMode.REUSE_DATASET_IF_EXISTS) - eval_dataset = MsDataset.load( - dataset_name='face_2d_keypoints_dataset', - namespace='modelscope', - split='train', - download_mode=DownloadMode.REUSE_DATASET_IF_EXISTS) - - kwargs = dict( - model=self.model_id, - train_dataset=train_dataset, - eval_dataset=eval_dataset, - work_dir=tmp_dir, - cfg_options=cfg_options) - - trainer = build_trainer(trainer_name, kwargs) - trainer.train() - - @unittest.skip( - 'skip since face_2d_keypoints_dataset is set to private for now') - def test_trainer_single_gpu(self): - temp_file_dir = tempfile.TemporaryDirectory() - tmp_dir = temp_file_dir.name - if not os.path.exists(tmp_dir): - os.makedirs(tmp_dir) - - self._train(tmp_dir) - - results_files = os.listdir(tmp_dir) - json_files = glob.glob(os.path.join(tmp_dir, '*.log.json')) - self.assertEqual(len(json_files), 1) - self.assertIn(f'{LogKeys.EPOCH}_2.pth', results_files) - - temp_file_dir.cleanup() - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/trainers/easycv/test_easycv_trainer_hand_2d_keypoints.py b/tests/trainers/easycv/test_easycv_trainer_hand_2d_keypoints.py deleted file mode 100644 index 270ecbc4..00000000 --- a/tests/trainers/easycv/test_easycv_trainer_hand_2d_keypoints.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import glob -import os -import shutil -import tempfile -import unittest - -import torch - -from modelscope.metainfo import Trainers -from modelscope.msdatasets import MsDataset -from modelscope.trainers import build_trainer -from modelscope.utils.constant import DownloadMode, LogKeys, Tasks -from modelscope.utils.logger import get_logger -from modelscope.utils.test_utils import test_level - - -@unittest.skipIf(not torch.cuda.is_available(), 'cuda unittest') -class EasyCVTrainerTestHand2dKeypoints(unittest.TestCase): - model_id = 'damo/cv_hrnetw18_hand-pose-keypoints_coco-wholebody' - - def setUp(self): - self.logger = get_logger() - self.logger.info(('Testing %s.%s' % - (type(self).__name__, self._testMethodName))) - self.tmp_dir = tempfile.TemporaryDirectory().name - if not os.path.exists(self.tmp_dir): - os.makedirs(self.tmp_dir) - - def tearDown(self): - super().tearDown() - shutil.rmtree(self.tmp_dir, ignore_errors=True) - - def _train(self): - cfg_options = {'train.max_epochs': 20} - - trainer_name = Trainers.easycv - - train_dataset = MsDataset.load( - dataset_name='cv_hand_2d_keypoints_coco_wholebody', - namespace='chenhyer', - split='subtrain', - download_mode=DownloadMode.FORCE_REDOWNLOAD) - eval_dataset = MsDataset.load( - dataset_name='cv_hand_2d_keypoints_coco_wholebody', - namespace='chenhyer', - split='subtrain', - download_mode=DownloadMode.FORCE_REDOWNLOAD) - - kwargs = dict( - model=self.model_id, - train_dataset=train_dataset, - eval_dataset=eval_dataset, - work_dir=self.tmp_dir, - cfg_options=cfg_options) - - trainer = build_trainer(trainer_name, kwargs) - trainer.train() - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_trainer_single_gpu(self): - self._train() - - results_files = os.listdir(self.tmp_dir) - json_files = glob.glob(os.path.join(self.tmp_dir, '*.log.json')) - self.assertEqual(len(json_files), 1) - self.assertIn(f'{LogKeys.EPOCH}_10.pth', results_files) - self.assertIn(f'{LogKeys.EPOCH}_20.pth', results_files) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/trainers/easycv/test_easycv_trainer_hand_detection.py b/tests/trainers/easycv/test_easycv_trainer_hand_detection.py deleted file mode 100644 index 60ea1319..00000000 --- a/tests/trainers/easycv/test_easycv_trainer_hand_detection.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import glob -import os -import shutil -import tempfile -import unittest - -import torch - -from modelscope.metainfo import Trainers -from modelscope.msdatasets import MsDataset -from modelscope.trainers import build_trainer -from modelscope.utils.constant import DownloadMode, LogKeys, Tasks -from modelscope.utils.logger import get_logger -from modelscope.utils.test_utils import test_level - - -class EasyCVTrainerTestHandDetection(unittest.TestCase): - model_id = 'damo/cv_yolox-pai_hand-detection' - - def setUp(self): - self.logger = get_logger() - self.logger.info(('Testing %s.%s' % - (type(self).__name__, self._testMethodName))) - - def _train(self, tmp_dir): - cfg_options = {'train.max_epochs': 2} - - trainer_name = Trainers.easycv - - train_dataset = MsDataset.load( - dataset_name='hand_detection_dataset', split='subtrain') - eval_dataset = MsDataset.load( - dataset_name='hand_detection_dataset', split='subtrain') - - kwargs = dict( - model=self.model_id, - train_dataset=train_dataset, - eval_dataset=eval_dataset, - work_dir=tmp_dir, - cfg_options=cfg_options) - - trainer = build_trainer(trainer_name, kwargs) - trainer.train() - - @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') - def test_trainer_single_gpu(self): - temp_file_dir = tempfile.TemporaryDirectory() - tmp_dir = temp_file_dir.name - if not os.path.exists(tmp_dir): - os.makedirs(tmp_dir) - - self._train(tmp_dir) - - results_files = os.listdir(tmp_dir) - # json_files = glob.glob(os.path.join(tmp_dir, '*.log.json')) - self.assertIn(f'{LogKeys.EPOCH}_2.pth', results_files) - - temp_file_dir.cleanup() - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/trainers/easycv/test_easycv_trainer_panoptic_mask2former.py b/tests/trainers/easycv/test_easycv_trainer_panoptic_mask2former.py deleted file mode 100644 index f6a6c41a..00000000 --- a/tests/trainers/easycv/test_easycv_trainer_panoptic_mask2former.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import glob -import os -import shutil -import tempfile -import unittest - -import torch -from mmcv.runner.hooks import HOOKS as MMCV_HOOKS - -from modelscope.metainfo import Trainers -from modelscope.msdatasets import MsDataset -from modelscope.trainers import build_trainer -from modelscope.utils.constant import LogKeys, Tasks -from modelscope.utils.logger import get_logger -from modelscope.utils.test_utils import test_level - - -@unittest.skipIf(not torch.cuda.is_available(), 'cuda unittest') -class EasyCVTrainerTestPanopticMask2Former(unittest.TestCase): - - def setUp(self): - self.logger = get_logger() - self.logger.info(('Testing %s.%s' % - (type(self).__name__, self._testMethodName))) - self.tmp_dir = tempfile.TemporaryDirectory().name - if not os.path.exists(self.tmp_dir): - os.makedirs(self.tmp_dir) - - def tearDown(self): - super().tearDown() - shutil.rmtree(self.tmp_dir, ignore_errors=True) - - def _train(self): - cfg_options = {'train.max_epochs': 1} - - trainer_name = Trainers.easycv - - train_dataset = MsDataset.load( - dataset_name='COCO2017_panopic_subset', split='train') - eval_dataset = MsDataset.load( - dataset_name='COCO2017_panopic_subset', split='validation') - kwargs = dict( - model='damo/cv_r50_panoptic-segmentation_cocopan', - train_dataset=train_dataset, - eval_dataset=eval_dataset, - work_dir=self.tmp_dir, - cfg_options=cfg_options) - - trainer = build_trainer(trainer_name, kwargs) - - hook_name = 'YOLOXLrUpdaterHook' - mmcv_hook = MMCV_HOOKS._module_dict.pop(hook_name, None) - - trainer.train() - - MMCV_HOOKS._module_dict[hook_name] = mmcv_hook - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_single_gpu_mask2former_r50(self): - self._train() - - results_files = os.listdir(self.tmp_dir) - json_files = glob.glob(os.path.join(self.tmp_dir, '*.log.json')) - self.assertEqual(len(json_files), 1) - self.assertIn(f'{LogKeys.EPOCH}_1.pth', results_files) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/trainers/easycv/test_easycv_trainer_realtime_object_detection.py b/tests/trainers/easycv/test_easycv_trainer_realtime_object_detection.py deleted file mode 100644 index 1171eed4..00000000 --- a/tests/trainers/easycv/test_easycv_trainer_realtime_object_detection.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import glob -import os -import shutil -import tempfile -import unittest - -import torch - -from modelscope.hub.snapshot_download import snapshot_download -from modelscope.metainfo import Trainers -from modelscope.msdatasets import MsDataset -from modelscope.trainers import build_trainer -from modelscope.utils.constant import DownloadMode, LogKeys, Tasks -from modelscope.utils.logger import get_logger -from modelscope.utils.test_utils import test_level - - -@unittest.skipIf(not torch.cuda.is_available(), 'cuda unittest') -class EasyCVTrainerTestRealtimeObjectDetection(unittest.TestCase): - model_id = 'damo/cv_cspnet_image-object-detection_yolox' - - def setUp(self): - self.logger = get_logger() - self.logger.info(('Testing %s.%s' % - (type(self).__name__, self._testMethodName))) - - def _train(self, tmp_dir): - # cfg_options = {'train.max_epochs': 2} - self.cache_path = snapshot_download(self.model_id) - cfg_options = { - 'train.max_epochs': - 2, - 'train.dataloader.batch_size_per_gpu': - 4, - 'evaluation.dataloader.batch_size_per_gpu': - 2, - 'train.hooks': [ - { - 'type': 'CheckpointHook', - 'interval': 1 - }, - { - 'type': 'EvaluationHook', - 'interval': 1 - }, - { - 'type': 'TextLoggerHook', - 'ignore_rounding_keys': None, - 'interval': 2 - }, - ], - 'load_from': - os.path.join(self.cache_path, 'pytorch_model.bin') - } - - trainer_name = Trainers.easycv - - train_dataset = MsDataset.load( - dataset_name='small_coco_for_test', - namespace='EasyCV', - split='train') - eval_dataset = MsDataset.load( - dataset_name='small_coco_for_test', - namespace='EasyCV', - split='validation') - - kwargs = dict( - model=self.model_id, - # model_revision='v1.0.2', - train_dataset=train_dataset, - eval_dataset=eval_dataset, - work_dir=tmp_dir, - cfg_options=cfg_options) - - trainer = build_trainer(trainer_name, kwargs) - trainer.train() - - @unittest.skipUnless( - test_level() >= 0, - 'skip since face_2d_keypoints_dataset is set to private for now') - def test_trainer_single_gpu(self): - temp_file_dir = tempfile.TemporaryDirectory() - tmp_dir = temp_file_dir.name - if not os.path.exists(tmp_dir): - os.makedirs(tmp_dir) - - self._train(tmp_dir) - - results_files = os.listdir(tmp_dir) - json_files = glob.glob(os.path.join(tmp_dir, '*.log.json')) - self.assertEqual(len(json_files), 1) - self.assertIn(f'{LogKeys.EPOCH}_2.pth', results_files) - - temp_file_dir.cleanup() - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/trainers/easycv/test_segformer.py b/tests/trainers/easycv/test_segformer.py deleted file mode 100644 index 90a66635..00000000 --- a/tests/trainers/easycv/test_segformer.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import glob -import os -import shutil -import tempfile -import unittest - -import torch - -from modelscope.metainfo import Trainers -from modelscope.msdatasets import MsDataset -from modelscope.trainers import build_trainer -from modelscope.utils.constant import LogKeys, Tasks -from modelscope.utils.logger import get_logger -from modelscope.utils.test_utils import test_level - - -@unittest.skipIf(not torch.cuda.is_available(), 'cuda unittest') -class EasyCVTrainerTestSegformer(unittest.TestCase): - - def setUp(self): - self.logger = get_logger() - self.logger.info(('Testing %s.%s' % - (type(self).__name__, self._testMethodName))) - self.tmp_dir = tempfile.TemporaryDirectory().name - if not os.path.exists(self.tmp_dir): - os.makedirs(self.tmp_dir) - - def tearDown(self): - super().tearDown() - shutil.rmtree(self.tmp_dir, ignore_errors=True) - - def _train(self): - - cfg_options = { - 'train.max_epochs': 2, - 'model.decode_head.norm_cfg.type': 'BN' - } - - trainer_name = Trainers.easycv - train_dataset = MsDataset.load( - dataset_name='small_coco_stuff164k', - namespace='EasyCV', - split='train') - eval_dataset = MsDataset.load( - dataset_name='small_coco_stuff164k', - namespace='EasyCV', - split='validation') - kwargs = dict( - model= - 'damo/cv_segformer-b0_image_semantic-segmentation_coco-stuff164k', - train_dataset=train_dataset, - eval_dataset=eval_dataset, - work_dir=self.tmp_dir, - cfg_options=cfg_options) - - trainer = build_trainer(trainer_name, kwargs) - trainer.train() - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_single_gpu_segformer(self): - self._train() - - results_files = os.listdir(self.tmp_dir) - json_files = glob.glob(os.path.join(self.tmp_dir, '*.log.json')) - self.assertEqual(len(json_files), 1) - self.assertIn(f'{LogKeys.EPOCH}_1.pth', results_files) - self.assertIn(f'{LogKeys.EPOCH}_2.pth', results_files) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/trainers/model_trainer_map.py b/tests/trainers/model_trainer_map.py index 4057c331..d4c4c09a 100644 --- a/tests/trainers/model_trainer_map.py +++ b/tests/trainers/model_trainer_map.py @@ -11,33 +11,18 @@ model_trainer_map = { ['tests/trainers/audio/test_separation_trainer.py'], 'speech_tts/speech_sambert-hifigan_tts_zh-cn_multisp_pretrain_16k': ['tests/trainers/audio/test_tts_trainer.py'], - 'damo/cv_mobilenet_face-2d-keypoints_alignment': - ['tests/trainers/easycv/test_easycv_trainer_face_2d_keypoints.py'], - 'damo/cv_hrnetw18_hand-pose-keypoints_coco-wholebody': - ['tests/trainers/easycv/test_easycv_trainer_hand_2d_keypoints.py'], - 'damo/cv_yolox-pai_hand-detection': - ['tests/trainers/easycv/test_easycv_trainer_hand_detection.py'], - 'damo/cv_r50_panoptic-segmentation_cocopan': - ['tests/trainers/easycv/test_easycv_trainer_panoptic_mask2former.py'], - 'damo/cv_segformer-b0_image_semantic-segmentation_coco-stuff164k': - ['tests/trainers/easycv/test_segformer.py'], 'damo/cv_resnet_carddetection_scrfd34gkps': ['tests/trainers/test_card_detection_scrfd_trainer.py'], - 'damo/multi-modal_clip-vit-base-patch16_zh': [ - 'tests/trainers/test_clip_trainer.py' - ], - 'damo/nlp_space_pretrained-dialog-model': [ - 'tests/trainers/test_dialog_intent_trainer.py' - ], - 'damo/cv_resnet_facedetection_scrfd10gkps': [ - 'tests/trainers/test_face_detection_scrfd_trainer.py' - ], - 'damo/nlp_structbert_faq-question-answering_chinese-base': [ - 'tests/trainers/test_finetune_faq_question_answering.py' - ], - 'PAI/nlp_gpt3_text-generation_0.35B_MoE-64': [ - 'tests/trainers/test_finetune_gpt_moe.py' - ], + 'damo/multi-modal_clip-vit-base-patch16_zh': + ['tests/trainers/test_clip_trainer.py'], + 'damo/nlp_space_pretrained-dialog-model': + ['tests/trainers/test_dialog_intent_trainer.py'], + 'damo/cv_resnet_facedetection_scrfd10gkps': + ['tests/trainers/test_face_detection_scrfd_trainer.py'], + 'damo/nlp_structbert_faq-question-answering_chinese-base': + ['tests/trainers/test_finetune_faq_question_answering.py'], + 'PAI/nlp_gpt3_text-generation_0.35B_MoE-64': + ['tests/trainers/test_finetune_gpt_moe.py'], 'damo/nlp_gpt3_text-generation_1.3B': [ 'tests/trainers/test_finetune_gpt3.py' ], From bdd282a12796e32459898cd06a87e4b425c88718 Mon Sep 17 00:00:00 2001 From: ly261666 Date: Wed, 10 May 2023 01:04:10 +0800 Subject: [PATCH 16/76] [to #42322933] add transface model Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12568301 --- data/test | 2 +- .../test_face_recognition_onnx_transface.py | 40 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 tests/pipelines/test_face_recognition_onnx_transface.py diff --git a/data/test b/data/test index 0a61e00d..8d062525 160000 --- a/data/test +++ b/data/test @@ -1 +1 @@ -Subproject commit 0a61e00de4a4b529099b357cbb0b2af83ac2f31e +Subproject commit 8d0625256b88bdf41655563049a4a68ec1025638 diff --git a/tests/pipelines/test_face_recognition_onnx_transface.py b/tests/pipelines/test_face_recognition_onnx_transface.py new file mode 100644 index 00000000..a41271c1 --- /dev/null +++ b/tests/pipelines/test_face_recognition_onnx_transface.py @@ -0,0 +1,40 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import unittest + +import numpy as np + +from modelscope.outputs import OutputKeys +from modelscope.pipelines import pipeline +from modelscope.utils.constant import Tasks +from modelscope.utils.demo_utils import DemoCompatibilityCheck +from modelscope.utils.test_utils import test_level + + +class TransFaceRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): + + def setUp(self) -> None: + self.task = Tasks.face_recognition + self.model_id = 'damo/cv_vit_face-recognition' + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_face_compare(self): + img1 = 'data/test/images/face_recognition_1.png' + img2 = 'data/test/images/face_recognition_2.png' + + face_recognition = pipeline( + Tasks.face_recognition, model=self.model_id) + emb1 = face_recognition(img1)[OutputKeys.IMG_EMBEDDING] + emb2 = face_recognition(img2)[OutputKeys.IMG_EMBEDDING] + if emb1 is None or emb2 is None: + print('No Detected Face.') + else: + sim = np.dot(emb1[0], emb2[0]) + print(f'Cos similarity={sim:.3f}, img1:{img1} img2:{img2}') + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_demo_compatibility(self): + self.compatibility_check() + + +if __name__ == '__main__': + unittest.main() From b80c13036f03e82573c0fc34002a8693108781ae Mon Sep 17 00:00:00 2001 From: "hehong.chh" Date: Wed, 10 May 2023 11:10:31 +0800 Subject: [PATCH 17/76] =?UTF-8?q?fix=20bug:=20hypotheses=20did't=20init=20?= =?UTF-8?q?in=20cpu=20device=E3=80=81make=20fid=5Fdialogue=5Ftest=20availa?= =?UTF-8?q?ble=20Link:=20https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/co?= =?UTF-8?q?dereview/12406082?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modelscope/pipelines/nlp/fid_dialogue_pipeline.py | 3 +-- tests/pipelines/test_fid_dialogue.py | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/modelscope/pipelines/nlp/fid_dialogue_pipeline.py b/modelscope/pipelines/nlp/fid_dialogue_pipeline.py index 702f303a..91815de7 100644 --- a/modelscope/pipelines/nlp/fid_dialogue_pipeline.py +++ b/modelscope/pipelines/nlp/fid_dialogue_pipeline.py @@ -191,8 +191,7 @@ class FidDialoguePipeline(Pipeline): def postprocess(self, inputs: TokenGeneratorOutput, **postprocess_params) -> Dict[str, Any]: - if torch.cuda.is_available(): - hypotheses = inputs.sequences.detach().cpu().tolist() + hypotheses = inputs.sequences.detach().cpu().tolist() response = self.preprocessor_tokenizer.decode( hypotheses[0], skip_special_tokens=self.is_t5) diff --git a/tests/pipelines/test_fid_dialogue.py b/tests/pipelines/test_fid_dialogue.py index c69823ce..600501ef 100644 --- a/tests/pipelines/test_fid_dialogue.py +++ b/tests/pipelines/test_fid_dialogue.py @@ -49,8 +49,7 @@ class FidDialogueTest(unittest.TestCase, DemoCompatibilityCheck): 'forward_params': forward_params } - # @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') - @unittest.skip('temporarily skip') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_240m_pipeline(self): pipeline_ins = pipeline( task=self.task, @@ -59,8 +58,7 @@ class FidDialogueTest(unittest.TestCase, DemoCompatibilityCheck): result = pipeline_ins(self.input, **self.kwargs) print(result) - # @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') - @unittest.skip('temporarily skip') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_3_7b_pipeline(self): pipeline_ins = pipeline( task=self.task, From ba646ef1d22a297970d3d1813ad0692f17338fc8 Mon Sep 17 00:00:00 2001 From: "mulin.lyh" Date: Wed, 10 May 2023 20:58:44 +0800 Subject: [PATCH 18/76] display hub error message Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12581373 --- modelscope/hub/errors.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/modelscope/hub/errors.py b/modelscope/hub/errors.py index be94d7fd..4bf2f935 100644 --- a/modelscope/hub/errors.py +++ b/modelscope/hub/errors.py @@ -2,6 +2,7 @@ from http import HTTPStatus +import requests from requests.exceptions import HTTPError from modelscope.utils.logger import get_logger @@ -57,13 +58,22 @@ def is_ok(rsp): return rsp['Code'] == HTTPStatus.OK and rsp['Success'] +def _decode_response_error(response: requests.Response): + if 'application/json' in response.headers.get('content-type', ''): + message = response.json() + else: + message = response.content.decode('utf-8') + return message + + def handle_http_post_error(response, url, request_body): try: response.raise_for_status() except HTTPError as error: logger.error('Request %s with body: %s exception' % (url, request_body)) - logger.error('Response details: %s' % response.content) + message = _decode_response_error(response) + logger.error('Response details: %s' % message) raise error @@ -75,7 +85,8 @@ def handle_http_response(response, logger, cookies, model_id): logger.error( f'Authentication token does not exist, failed to access model {model_id} which may not exist or may be \ private. Please login first.') - logger.error('Response details: %s' % response.content) + message = _decode_response_error(response) + logger.error('Response details: %s' % message) raise error From 7b14a0e11f9ed0ce08490e8ee0c18f2b23e6e58a Mon Sep 17 00:00:00 2001 From: "mulin.lyh" Date: Thu, 11 May 2023 11:20:01 +0800 Subject: [PATCH 19/76] Pipeline input, output and parameter normalization. --- modelscope/metainfo.py | 1 + modelscope/outputs/outputs.py | 349 +++++++- modelscope/pipeline_inputs.py | 69 +- .../text_to_video_synthesis_pipeline.py | 13 +- .../pipelines/nlp/fid_dialogue_pipeline.py | 5 +- .../nlp/table_question_answering_pipeline.py | 2 +- .../nlp/text_classification_pipeline.py | 2 +- modelscope/pipelines/pipeline_template.py | 87 ++ .../nlp/token_classification_preprocessor.py | 2 +- modelscope/utils/constant.py | 1 + modelscope/utils/input_output.py | 756 ++++++++++++++++++ .../test_table_question_answering.py | 8 +- tests/utils/test_input_output.py | 142 ++++ 13 files changed, 1415 insertions(+), 22 deletions(-) create mode 100644 modelscope/pipelines/pipeline_template.py create mode 100644 modelscope/utils/input_output.py create mode 100644 tests/utils/test_input_output.py diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index 8d6034e2..0d7b5f64 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -250,6 +250,7 @@ class Pipelines(object): should use task name for this pipeline. For pipeline which suuport only one model, we should use ${Model}-${Task} as its name. """ + pipeline_template = 'pipeline-template' # vision tasks portrait_matting = 'unet-image-matting' universal_matting = 'unet-universal-matting' diff --git a/modelscope/outputs/outputs.py b/modelscope/outputs/outputs.py index ddbe4593..1cf891c7 100644 --- a/modelscope/outputs/outputs.py +++ b/modelscope/outputs/outputs.py @@ -1,6 +1,10 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from collections import OrderedDict, namedtuple from dataclasses import dataclass, fields +from typing import Dict, List, Tuple + +import numpy as np +import torch from modelscope.utils.constant import Tasks @@ -50,7 +54,7 @@ class OutputKeys(object): SQL_STRING = 'sql_string' SQL_QUERY = 'sql_query' HISTORY = 'history' - QUERT_RESULT = 'query_result' + QUERY_RESULT = 'query_result' TIMESTAMPS = 'timestamps' SHOT_NUM = 'shot_num' SCENE_NUM = 'scene_num' @@ -62,8 +66,343 @@ class OutputKeys(object): TBOUNDS = 'tbounds' -TASK_OUTPUTS = { +OutputTypes = { + OutputKeys.LOSS: float, # checked + OutputKeys.LOGITS: np.ndarray, # checked. + OutputKeys.SCORES: List[float], # checked + OutputKeys.SCORE: float, # checked + OutputKeys.LABEL: str, # checked + OutputKeys.LABELS: List[str], # checked + OutputKeys.INPUT_IDS: np.ndarray, # checked + OutputKeys.LABEL_POS: np.ndarray, # checked + OutputKeys.POSES: + List[np.ndarray], # [Tuple(np.ndarray, np.ndarray)] # checked doubtful + OutputKeys.CAPTION: str, + OutputKeys.BOXES: np.ndarray, # checked + OutputKeys.KEYPOINTS: np.ndarray, # checked + OutputKeys.MASKS: np.ndarray, # checked + OutputKeys.DEPTHS: List[np.ndarray], # checked + OutputKeys.DEPTHS_COLOR: List[np.ndarray], # checked + OutputKeys.LAYOUT: np.ndarray, # checked + OutputKeys.TEXT: str, # checked + OutputKeys.POLYGONS: np.array, # checked + OutputKeys.OUTPUT: Dict, + OutputKeys.OUTPUT_IMG: 'image', # checked + OutputKeys.OUTPUT_IMGS: List[np.ndarray], # checked + OutputKeys.OUTPUT_VIDEO: 'bytes', + OutputKeys.OUTPUT_PCM: np.ndarray, + OutputKeys.OUTPUT_PCM_LIST: List[np.ndarray], + OutputKeys.OUTPUT_WAV: np.ndarray, + OutputKeys.OUTPUT_OBJ: Dict, + OutputKeys.OUTPUT_MESH: np.ndarray, + OutputKeys.IMG_EMBEDDING: np.ndarray, + OutputKeys.SPK_EMBEDDING: np.ndarray, + OutputKeys.SPO_LIST: List[float], + OutputKeys.TEXT_EMBEDDING: np.ndarray, + OutputKeys.TRANSLATION: str, + OutputKeys.RESPONSE: Dict, + OutputKeys.PREDICTION: np.ndarray, # checked + OutputKeys.PREDICTIONS: List[np.ndarray], + OutputKeys.PROBABILITIES: np.ndarray, + OutputKeys.DIALOG_STATES: object, + OutputKeys.VIDEO_EMBEDDING: np.ndarray, + OutputKeys.UUID: str, + OutputKeys.WORD: str, + OutputKeys.KWS_LIST: List[str], + OutputKeys.SQL_STRING: str, # checked + OutputKeys.SQL_QUERY: str, # checked + OutputKeys.HISTORY: Dict, # checked + OutputKeys.QUERY_RESULT: Dict, # checked + OutputKeys.TIMESTAMPS: str, + OutputKeys.SHOT_NUM: int, + OutputKeys.SCENE_NUM: int, + OutputKeys.SCENE_META_LIST: List[int], + OutputKeys.SHOT_META_LIST: List[int], + OutputKeys.MATCHES: List[np.ndarray], + OutputKeys.PCD12: np.ndarray, + OutputKeys.PCD12_ALIGN: np.ndarray, + OutputKeys.TBOUNDS: Dict, +} +OutputTypeSchema = { + OutputKeys.LOSS: { + 'type': 'number' + }, # checked + OutputKeys.LOGITS: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, # checked. + OutputKeys.SCORES: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, # checked + OutputKeys.SCORE: { + 'type': 'number' + }, # checked + OutputKeys.LABEL: { + 'type': 'string' + }, # checked + OutputKeys.LABELS: { + 'type': 'array', + 'items': { + 'type': 'string' + } + }, # checked + OutputKeys.INPUT_IDS: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, # checked + OutputKeys.LABEL_POS: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, # checked + OutputKeys.POSES: { + 'type': 'array', + 'items': { + 'type': 'array', + 'items': { + 'type': 'number' + } + } + }, # [Tuple(np.ndarray, np.ndarray)] # checked doubtful + OutputKeys.CAPTION: { + 'type': 'string' + }, + OutputKeys.BOXES: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, # checked + OutputKeys.KEYPOINTS: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, # checked + OutputKeys.MASKS: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, # checked + OutputKeys.DEPTHS: { + 'type': 'array', + 'items': { + 'type': 'array', + 'items': { + 'type': 'number' + } + } + }, # checked + OutputKeys.DEPTHS_COLOR: { + 'type': 'array', + 'items': { + 'type': 'array', + 'items': { + 'type': 'number' + } + } + }, # checked + OutputKeys.LAYOUT: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, # checked + OutputKeys.TEXT: { + 'type': 'string' + }, # checked + OutputKeys.POLYGONS: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, # checked + OutputKeys.OUTPUT: { + 'type': 'object' + }, + OutputKeys.OUTPUT_IMG: { + 'type': 'string', + 'description': 'The base64 encoded image.', + }, # checked + OutputKeys.OUTPUT_IMGS: { + 'type': 'array', + 'items': { + 'type': 'string', + 'description': 'The base64 encoded image.', + } + }, # checked + OutputKeys.OUTPUT_VIDEO: { + 'type': 'string', + 'description': 'The base64 encoded video.', + }, + OutputKeys.OUTPUT_PCM: { + 'type': 'string', + 'description': 'The base64 encoded PCM.', + }, + OutputKeys.OUTPUT_PCM_LIST: { + 'type': 'array', + 'items': { + 'type': 'string', + 'description': 'The base64 encoded PCM.', + } + }, + OutputKeys.OUTPUT_WAV: { + 'type': 'string', + 'description': 'The base64 encoded WAV.', + }, + OutputKeys.OUTPUT_OBJ: { + 'type': 'object' + }, + OutputKeys.OUTPUT_MESH: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, + OutputKeys.IMG_EMBEDDING: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, + OutputKeys.SPK_EMBEDDING: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, + OutputKeys.SPO_LIST: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, + OutputKeys.TEXT_EMBEDDING: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, + OutputKeys.TRANSLATION: { + 'type': 'string' + }, + OutputKeys.RESPONSE: { + 'type': 'object' + }, + OutputKeys.PREDICTION: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, # checked + OutputKeys.PREDICTIONS: { + 'type': 'array', + 'items': { + 'type': 'array', + 'items': { + 'type': 'number' + } + } + }, + OutputKeys.PROBABILITIES: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, + OutputKeys.DIALOG_STATES: { + 'type': 'object' + }, + OutputKeys.VIDEO_EMBEDDING: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, + OutputKeys.UUID: { + 'type': 'string' + }, + OutputKeys.WORD: { + 'type': 'string' + }, + OutputKeys.KWS_LIST: { + 'type': 'array', + 'items': { + 'type': 'string' + } + }, + OutputKeys.SQL_STRING: { + 'type': 'string' + }, # checked + OutputKeys.SQL_QUERY: { + 'type': 'string' + }, # checked + OutputKeys.HISTORY: { + 'type': 'object' + }, # checked + OutputKeys.QUERY_RESULT: { + 'type': 'object' + }, # checked + OutputKeys.TIMESTAMPS: { + 'type': 'string' + }, + OutputKeys.SHOT_NUM: { + 'type': 'integer' + }, + OutputKeys.SCENE_NUM: { + 'type': 'integer' + }, + OutputKeys.SCENE_META_LIST: { + 'type': 'array', + 'items': { + 'type': 'integer' + } + }, + OutputKeys.SHOT_META_LIST: { + 'type': 'array', + 'items': { + 'type': 'integer' + } + }, + OutputKeys.MATCHES: { + 'type': 'array', + 'items': { + 'type': 'array', + 'items': { + 'type': 'number' + } + } + }, + OutputKeys.PCD12: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, + OutputKeys.PCD12_ALIGN: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, + OutputKeys.TBOUNDS: { + 'type': 'object' + }, +} + +TASK_OUTPUTS = { + Tasks.task_template: + [OutputKeys.BOXES, OutputKeys.OUTPUT_IMG, OutputKeys.TEXT_EMBEDDING], # ============ vision tasks =================== # ocr detection result for single sample @@ -388,8 +727,9 @@ TASK_OUTPUTS = { # "scores": [0.885272, 0.014790631, 0.014558001] # "labels": ['噪声强度', '模糊程度', '压缩强度'], # } - Tasks.image_quality_assessment_degradation: - [OutputKeys.SCORES, OutputKeys.LABELS], + Tasks.image_quality_assessment_degradation: [ + OutputKeys.SCORES, OutputKeys.LABELS + ], # live category recognition result for single video # { @@ -1140,6 +1480,7 @@ TASK_OUTPUTS = { Tasks.document_grounded_dialog_rerank: [OutputKeys.OUTPUT], Tasks.document_grounded_dialog_retrieval: [OutputKeys.OUTPUT], Tasks.video_temporal_grounding: [OutputKeys.SCORES, OutputKeys.TBOUNDS], + Tasks.text_to_video_synthesis: [OutputKeys.OUTPUT_VIDEO], } diff --git a/modelscope/pipeline_inputs.py b/modelscope/pipeline_inputs.py index 032bdff6..29c5b8d8 100644 --- a/modelscope/pipeline_inputs.py +++ b/modelscope/pipeline_inputs.py @@ -20,7 +20,7 @@ class InputType(object): BOX = 'box' DICT = 'dict' LIST = 'list' - INT = 'int' + NUMBER = 'number' INPUT_TYPE = { @@ -31,7 +31,42 @@ INPUT_TYPE = { InputType.BOX: (list, np.ndarray), InputType.DICT: (dict, type(None)), InputType.LIST: (list, type(None)), - InputType.INT: int, + InputType.NUMBER: int, +} + +INPUT_TYPE_SCHEMA = { + InputType.IMAGE: { + 'type': 'string', + 'description': 'Base64 encoded image file or url string.' + }, # support url or base64 encoded file. + InputType.AUDIO: { + 'type': 'string', + 'description': 'Base64 encoded audio file or url string..' + }, # support url or base64 encoded file. + InputType.VIDEO: { + 'type': 'string', + 'description': 'Base64 encoded video file or url string..' + }, # support url or base64 encoded file. + InputType.TEXT: { + 'type': 'string', + 'description': 'The input text.' + }, + InputType.BOX: { + 'type': 'array', + 'description': 'Box coordinate, should be int.', + 'items': { + 'type': 'number' + } + }, + InputType.DICT: { # unknown properties + 'type': 'object', + }, + InputType.LIST: { + 'type': 'array' + }, # unknown item type. + InputType.NUMBER: { + 'type': 'integer' + }, } @@ -47,12 +82,19 @@ def check_input_type(input_type, input): TASK_INPUTS = { + + Tasks.task_template: { + 'image': InputType.IMAGE, + 'text': InputType.TEXT + }, # if task input is single var, value is InputType # if task input is a tuple, value is tuple of InputType # if task input is a dict, value is a dict of InputType, where key # equals the one needed in pipeline input dict # if task input is a list, value is a set of input format, in which - # each element corresponds to one input format as described above. + # each element corresponds to one input format as described above and + # must include a dict format. + # ============ vision tasks =================== Tasks.ocr_detection: InputType.IMAGE, @@ -73,7 +115,7 @@ TASK_INPUTS = { Tasks.human_detection: InputType.IMAGE, Tasks.face_image_generation: - InputType.INT, + InputType.NUMBER, Tasks.image_classification: InputType.IMAGE, Tasks.image_object_detection: @@ -191,8 +233,7 @@ TASK_INPUTS = { Tasks.nli: (InputType.TEXT, InputType.TEXT), Tasks.sentiment_classification: InputType.TEXT, - Tasks.zero_shot_classification: - InputType.TEXT, + Tasks.zero_shot_classification: InputType.TEXT, Tasks.relation_extraction: InputType.TEXT, Tasks.translation: @@ -212,7 +253,13 @@ TASK_INPUTS = { 'source_sentence': InputType.LIST, 'sentences_to_compare': InputType.LIST, }, - Tasks.text_ranking: (InputType.TEXT, InputType.TEXT), + Tasks.text_ranking: [ + (InputType.TEXT, InputType.TEXT), + { + 'source_sentence': InputType.LIST, + 'sentences_to_compare': InputType.LIST + } + ], Tasks.text_generation: InputType.TEXT, Tasks.fid_dialogue: { @@ -261,7 +308,7 @@ TASK_INPUTS = { }, # ============ audio tasks =================== - Tasks.auto_speech_recognition: + Tasks.auto_speech_recognition: # input can be audio, or audio and text. [InputType.AUDIO, { 'wav': InputType.AUDIO, 'text': InputType.TEXT @@ -332,5 +379,9 @@ TASK_INPUTS = { 'video_input_path': InputType.TEXT, 'video_output_path': InputType.TEXT, 'mask_path': InputType.TEXT, - } + }, + Tasks.text_to_video_synthesis: { + 'text': InputType.TEXT + }, + Tasks.video_summarization: InputType.TEXT, } diff --git a/modelscope/pipelines/multi_modal/text_to_video_synthesis_pipeline.py b/modelscope/pipelines/multi_modal/text_to_video_synthesis_pipeline.py index ee6635a6..50e2437b 100644 --- a/modelscope/pipelines/multi_modal/text_to_video_synthesis_pipeline.py +++ b/modelscope/pipelines/multi_modal/text_to_video_synthesis_pipeline.py @@ -1,5 +1,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. +import os import tempfile from typing import Any, Dict, Optional @@ -62,8 +63,10 @@ class TextToVideoSynthesisPipeline(Pipeline): **post_params) -> Dict[str, Any]: video = tensor2vid(inputs['video']) output_video_path = post_params.get('output_video', None) + temp_video_file = False if output_video_path is None: output_video_path = tempfile.NamedTemporaryFile(suffix='.mp4').name + temp_video_file = True fourcc = cv2.VideoWriter_fourcc(*'mp4v') h, w, c = video[0].shape @@ -72,7 +75,15 @@ class TextToVideoSynthesisPipeline(Pipeline): for i in range(len(video)): img = cv2.cvtColor(video[i], cv2.COLOR_RGB2BGR) video_writer.write(img) - return {OutputKeys.OUTPUT_VIDEO: output_video_path} + video_writer.release() + if temp_video_file: + video_file_content = b'' + with open(output_video_path, 'rb') as f: + video_file_content = f.read() + os.remove(output_video_path) + return {OutputKeys.OUTPUT_VIDEO: video_file_content} + else: + return {OutputKeys.OUTPUT_VIDEO: output_video_path} def tensor2vid(video, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]): diff --git a/modelscope/pipelines/nlp/fid_dialogue_pipeline.py b/modelscope/pipelines/nlp/fid_dialogue_pipeline.py index 91815de7..6e2cd2cf 100644 --- a/modelscope/pipelines/nlp/fid_dialogue_pipeline.py +++ b/modelscope/pipelines/nlp/fid_dialogue_pipeline.py @@ -191,7 +191,10 @@ class FidDialoguePipeline(Pipeline): def postprocess(self, inputs: TokenGeneratorOutput, **postprocess_params) -> Dict[str, Any]: - hypotheses = inputs.sequences.detach().cpu().tolist() + if torch.cuda.is_available(): + hypotheses = inputs.sequences.detach().cpu().tolist() + else: + hypotheses = inputs.sequences response = self.preprocessor_tokenizer.decode( hypotheses[0], skip_special_tokens=self.is_t5) diff --git a/modelscope/pipelines/nlp/table_question_answering_pipeline.py b/modelscope/pipelines/nlp/table_question_answering_pipeline.py index 365c6c6c..e1646e1d 100644 --- a/modelscope/pipelines/nlp/table_question_answering_pipeline.py +++ b/modelscope/pipelines/nlp/table_question_answering_pipeline.py @@ -401,7 +401,7 @@ class TableQuestionAnsweringPipeline(Pipeline): OutputKeys.SQL_STRING: sql.string, OutputKeys.SQL_QUERY: sql.query, OutputKeys.HISTORY: result['sql'], - OutputKeys.QUERT_RESULT: tabledata, + OutputKeys.QUERY_RESULT: tabledata, } return {OutputKeys.OUTPUT: output} diff --git a/modelscope/pipelines/nlp/text_classification_pipeline.py b/modelscope/pipelines/nlp/text_classification_pipeline.py index a300b008..3b06f435 100644 --- a/modelscope/pipelines/nlp/text_classification_pipeline.py +++ b/modelscope/pipelines/nlp/text_classification_pipeline.py @@ -76,7 +76,7 @@ class TextClassificationPipeline(Pipeline): field=Fields.multi_modal, **kwargs) else: - first_sequence = kwargs.pop('first_sequence', 'first_sequence') + first_sequence = kwargs.pop('first_sequence', 'text') second_sequence = kwargs.pop('second_sequence', None) sequence_length = kwargs.pop('sequence_length', 512) self.preprocessor = Preprocessor.from_pretrained( diff --git a/modelscope/pipelines/pipeline_template.py b/modelscope/pipelines/pipeline_template.py new file mode 100644 index 00000000..a29ce5d7 --- /dev/null +++ b/modelscope/pipelines/pipeline_template.py @@ -0,0 +1,87 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from typing import Any, Dict + +import numpy as np + +from modelscope.metainfo import Pipelines +from modelscope.models.base.base_model import Model +from modelscope.outputs.outputs import OutputKeys +from modelscope.pipelines.base import Pipeline +from modelscope.pipelines.builder import PIPELINES +from modelscope.utils.constant import Tasks + +__all__ = ['PipelineTemplate'] + + +@PIPELINES.register_module( + Tasks.task_template, module_name=Pipelines.pipeline_template) +class PipelineTemplate(Pipeline): + """A pipeline template explain how to define parameters and input and + output information. As a rule, the first parameter is the input, + followed by the request parameters. The parameter must add type + hint information, and set the default value if necessary, + for the convenience of use. + """ + + def __init__(self, model: Model, **kwargs): + """A pipeline template to describe input and + output and parameter processing + + Args: + model: A Model instance. + """ + # call base init. + super().__init__(model=model, **kwargs) + + def preprocess(self, + input: Any, + max_length: int = 1024, + top_p: float = 0.8) -> Any: + """Pipeline preprocess interface. + + Args: + input (Any): The pipeline input, ref Tasks.task_template TASK_INPUTS. + max_length (int, optional): The max_length parameter. Defaults to 1024. + top_p (float, optional): The top_p parameter. Defaults to 0.8. + + Returns: + Any: Return result process by forward. + """ + pass + + def forward(self, + input: Any, + max_length: int = 1024, + top_p: float = 0.8) -> Any: + """The forward interface. + + Args: + input (Any): The output of the preprocess. + max_length (int, optional): max_length. Defaults to 1024. + top_p (float, optional): top_p. Defaults to 0.8. + + Returns: + Any: Return result process by postprocess. + """ + pass + + def postprocess(self, + inputs: Any, + postprocess_param1: str = None) -> Dict[str, Any]: + """The postprocess interface. + + Args: + input (Any): The output of the forward. + max_length (int, optional): max_length. Defaults to 1024. + top_p (float, optional): top_p. Defaults to 0.8. + + Returns: + Any: Return result process by postprocess. + """ + result = { + OutputKeys.BOXES: np.zeros(4), + OutputKeys.OUTPUT_IMG: np.zeros(10, 4), + OutputKeys.TEXT_EMBEDDING: np.zeros(1, 1000) + } + return result diff --git a/modelscope/preprocessors/nlp/token_classification_preprocessor.py b/modelscope/preprocessors/nlp/token_classification_preprocessor.py index 66e57cc8..4b4fee1f 100644 --- a/modelscope/preprocessors/nlp/token_classification_preprocessor.py +++ b/modelscope/preprocessors/nlp/token_classification_preprocessor.py @@ -201,7 +201,7 @@ class TokenClassificationTransformersPreprocessor( def __init__(self, model_dir: str = None, - first_sequence: str = None, + first_sequence: str = 'text', label: str = 'label', label2id: Dict = None, label_all_tokens: bool = False, diff --git a/modelscope/utils/constant.py b/modelscope/utils/constant.py index a19eef20..5a23902f 100644 --- a/modelscope/utils/constant.py +++ b/modelscope/utils/constant.py @@ -277,6 +277,7 @@ class Tasks(CVTasks, NLPTasks, AudioTasks, MultiModalTasks, ScienceTasks): This should be used to register models, pipelines, trainers. """ reverse_field_index = {} + task_template = 'task-template' @staticmethod def find_field_by_task(task_name): diff --git a/modelscope/utils/input_output.py b/modelscope/utils/input_output.py new file mode 100644 index 00000000..b2c9cd5b --- /dev/null +++ b/modelscope/utils/input_output.py @@ -0,0 +1,756 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import ast +import base64 +import importlib +import inspect +from io import BytesIO +from typing import Any +from urllib.parse import urlparse + +import numpy as np + +from modelscope.hub.api import HubApi +from modelscope.hub.errors import NotExistError +from modelscope.hub.file_download import model_file_download +from modelscope.outputs.outputs import (TASK_OUTPUTS, OutputKeys, OutputTypes, + OutputTypeSchema) +from modelscope.pipeline_inputs import (INPUT_TYPE, INPUT_TYPE_SCHEMA, + TASK_INPUTS, InputType) +from modelscope.pipelines import pipeline +from modelscope.pipelines.base import Pipeline +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() +"""Support webservice integration pipeline。 + +This module provides a support library when webservice uses pipeline, +converts webservice input into pipeline input, and converts pipeline +output into webservice output, which automatically encodes and +decodes relevant fields. + +Example: + # create pipeine instance and pipeline information, save it to app + pipeline_instance = create_pipeline('damo/cv_gpen_image-portrait-enhancement', 'v1.0.0') + pipeline_info = get_pipeline_information_by_pipeline(pipeline_instance) + app.state.pipeline = pipeline_instance + app.state.pipeline_info = pipeline_info + + # for service schema request. + pipeline_info = request.app.state.pipeline_info + return pipeline_info.schema + + # for service call request. + def inference(request: Request): + pipeline_service = request.app.state.pipeline + pipeline_info = request.app.state.pipeline_info + request_json = await request.json() + result = call_pipeline_with_json(pipeline_info, + pipeline_service, + request_json) + # convert output to json, if binary field, we need encoded. + output = pipeline_output_to_service_base64_output(pipeline_info.task_name, result) + return output +Todo: + * Support more service input type, such as form. + +""" + + +def create_pipeline(model_id: str, revision: str): + model_configuration_file = model_file_download( + model_id=model_id, + file_path=ModelFile.CONFIGURATION, + revision=revision) + cfg = Config.from_file(model_configuration_file) + return pipeline(task=cfg.task, model=model_id, model_revision=revision) + + +def get_class_user_attributes(cls): + attributes = inspect.getmembers(cls, lambda a: not (inspect.isroutine(a))) + user_attributes = [ + a for a in attributes + if (not (a[0].startswith('__') and a[0].endswith('__'))) + ] + return user_attributes + + +def get_input_type(task_inputs: Any): + """Get task input schema. + + Args: + task_name (str): The task name. + """ + if isinstance(task_inputs, str): # no input key + input_type = INPUT_TYPE[task_inputs] + return input_type + elif isinstance(task_inputs, tuple) or isinstance(task_inputs, list): + for item in task_inputs: + if isinstance(item, + dict): # for list, server only support dict format. + return get_input_type(item) + else: + continue + elif isinstance(task_inputs, dict): + input_info = {} # key input key, value input type + for k, v in task_inputs.items(): + input_info[k] = get_input_type(v) + return input_info + else: + raise ValueError(f'invalid input_type definition {task_inputs}') + + +def get_input_schema(task_name: str, input_type: type): + """Get task input schema. + + Args: + task_name (str): The task name. + input_type (type): The input type + """ + if input_type is None: + task_inputs = TASK_INPUTS[task_name] + if isinstance(task_inputs, + str): # only one input field, key is task_inputs + return { + 'type': 'object', + 'properties': { + task_inputs: INPUT_TYPE_SCHEMA[task_inputs] + } + } + else: + task_inputs = input_type + + if isinstance(task_inputs, str): # no input key + return INPUT_TYPE_SCHEMA[task_inputs] + elif input_type is None and isinstance(task_inputs, list): + for item in task_inputs: + # for list, server only support dict format. + if isinstance(item, dict): + return get_input_schema(None, item) + elif isinstance(task_inputs, tuple) or isinstance(task_inputs, list): + input_schema = {'type': 'array', 'items': {}} + for item in task_inputs: + if isinstance(item, dict): + item_schema = get_input_schema(None, item) + input_schema['items']['type'] = item_schema + return input_schema + else: + input_schema['items'] = INPUT_TYPE_SCHEMA[item] + return input_schema + + elif isinstance(task_inputs, dict): + input_schema = { + 'type': 'object', + 'properties': {} + } # key input key, value input type + for k, v in task_inputs.items(): + input_schema['properties'][k] = get_input_schema(None, v) + return input_schema + else: + raise ValueError(f'invalid input_type definition {task_inputs}') + + +def get_output_schema(task_name: str): + """Get task output schema. + + Args: + task_name (str): The task name. + """ + task_outputs = TASK_OUTPUTS[task_name] + output_schema = {'type': 'object', 'properties': {}} + if not isinstance(task_outputs, list): + raise ValueError('TASK_OUTPUTS for %s is not list.' % task_name) + else: + for output_key in task_outputs: + output_schema['properties'][output_key] = OutputTypeSchema[ + output_key] + return output_schema + + +def get_input_info(task_name: str): + task_inputs = TASK_INPUTS[task_name] + if isinstance(task_inputs, str): # no input key default input key input + input_type = INPUT_TYPE[task_inputs] + return input_type + elif isinstance(task_inputs, tuple): + return task_inputs + elif isinstance(task_inputs, list): + for item in task_inputs: + if isinstance(item, + dict): # for list, server only support dict format. + return {'input': get_input_type(item)} + else: + continue + elif isinstance(task_inputs, dict): + input_info = {} # key input key, value input type + for k, v in task_inputs.items(): + input_info[k] = get_input_type(v) + return {'input': input_info} + else: + raise ValueError(f'invalid input_type definition {task_inputs}') + + +def get_output_info(task_name: str): + output_keys = TASK_OUTPUTS[task_name] + output_type = {} + if not isinstance(output_keys, list): + raise ValueError('TASK_OUTPUTS for %s is not list.' % task_name) + else: + for output_key in output_keys: + output_type[output_key] = OutputTypes[output_key] + return output_type + + +def get_task_io_info(task_name: str): + """Get task input output schema. + + Args: + task_name (str): The task name. + """ + tasks = get_class_user_attributes(Tasks) + task_exist = False + for key, value in tasks: + if key == task_name or value == task_name: + task_exist = True + break + if not task_exist: + return None, None + + task_inputs = get_input_info(task_name) + task_outputs = get_output_info(task_name) + + return task_inputs, task_outputs + + +def process_arg_type_annotation(arg, default_value): + if arg.annotation is not None: + if isinstance(arg.annotation, ast.Subscript): + return arg.arg, arg.annotation.value.id + elif isinstance(arg.annotation, ast.Name): + return arg.arg, arg.annotation.id + elif isinstance(arg.annotation, ast.Attribute): + return arg.arg, arg.annotation.attr + else: + raise Exception('Invalid annotation: %s' % arg.annotation) + else: + if default_value is not None: + return arg.arg, type(default_value).__name__ + # Irregular, assuming no type hint no default value type is object + logger.warning('arg: %s has no data type annotation, use default!' % + (arg.arg)) + return arg.arg, 'object' + + +def process_args(args): + arguments = [] + # name, type, has_default, default + n_args = len(args.args) + n_args_default = len(args.defaults) + # no default + for arg in args.args[0:n_args - n_args_default]: + if arg.arg == 'self': + continue + else: + arg_name, arg_type = process_arg_type_annotation(arg, None) + arguments.append((arg_name, arg_type, False, None)) + + # process defaults arg. + for arg, dft in zip(args.args[n_args - n_args_default:], args.defaults): + # compatible with python3.7 ast.Num no value. + value = dft.value if hasattr(dft, 'value') else dft.n + arg_name, arg_type = process_arg_type_annotation(arg, value) + arguments.append((arg_name, arg_type, True, value)) + + # kwargs + n_kwargs = len(args.kwonlyargs) + n_kwargs_default = len(args.kw_defaults) + for kwarg in args.kwonlyargs[0:n_kwargs - n_kwargs_default]: + arg_name, arg_type = process_arg_type_annotation(kwarg) + arguments.append((arg_name, arg_type, False, None)) + + for kwarg, dft in zip(args.kwonlyargs[n_kwargs - n_kwargs_default:], + args.kw_defaults): + arg_name, arg_type = process_arg_type_annotation(kwarg) + arguments.append((arg_name, arg_type, True, dft.value)) + return arguments + + +class PipelineClassAnalyzer(ast.NodeVisitor): + """Analysis pipeline class define get inputs and parameters. + """ + + def __init__(self) -> None: + super().__init__() + self.parameters = [] + self.has_call = False + self.preprocess_parameters = [] + self.has_preprocess = False + self.has_postprocess = False + self.has_forward = False + self.forward_parameters = [] + self.postprocess_parameters = [] + self.lineno = 0 + self.end_lineno = 0 + + def visit_FunctionDef(self, node: ast.FunctionDef) -> Any: + if node.name == '__call__': + self.parameters = process_args(node.args) + self.has_call = True + if node.name == 'preprocess': + self.preprocess_parameters = process_args(node.args) + self.has_preprocess = True + elif node.name == 'postprocess': + self.postprocess_parameters = process_args(node.args) + self.has_postprocess = True + elif node.name == 'forward': + self.forward_parameters = process_args(node.args) + self.has_forward = True + + def get_input_parameters(self): + if self.has_call: + # custom define __call__ inputs and parameter are control by the + # custom __call__, all parameter is input. + return self.parameters, None + parameters = [] + if self.has_preprocess: + parameters.extend(self.preprocess_parameters[1:]) + if self.has_forward: + parameters.extend(self.forward_parameters[1:]) + if self.has_postprocess: + parameters.extend(self.postprocess_parameters[1:]) + + if len(parameters) > 0: + return None, parameters + else: + return None, [] + + +class AnalysisSourceFileRegisterModules(ast.NodeVisitor): + """Get register_module call of the python source file. + + + Args: + ast (NodeVisitor): The ast node. + + Examples: + >>> with open(source_file_path, "rb") as f: + >>> src = f.read() + >>> analyzer = AnalysisSourceFileRegisterModules(source_file_path) + >>> analyzer.visit(ast.parse(src, filename=source_file_path)) + """ + + def __init__(self, source_file_path, class_name) -> None: + super().__init__() + self.source_file_path = source_file_path + self.class_name = class_name + self.class_define = None + + def visit_ClassDef(self, node: ast.ClassDef): + if node.name == self.class_name: + self.class_define = node + + +def get_pipeline_input_parameters( + source_file_path: str, + class_name: str, +): + """Get pipeline input and parameter + + Args: + source_file_path (str): The pipeline source code path + class_name (str): The pipeline class name + """ + with open(source_file_path, 'rb') as f: + src = f.read() + analyzer = AnalysisSourceFileRegisterModules(source_file_path, + class_name) + analyzer.visit( + ast.parse( + src, + filename=source_file_path, + # python3.7 no type_comments parameter . + # type_comments=True + )) + clz = PipelineClassAnalyzer() + clz.visit(analyzer.class_define) + input, pipeline_parameters = clz.get_input_parameters() + # remove the first input parameter, the input is defined by task. + return input, pipeline_parameters + + +meta_type_schema_map = { + # For parameters, current only support types. + 'str': 'string', + 'int': 'integer', + 'float': 'number', + 'bool': 'boolean', + 'Dict': 'object', + 'dict': 'object', + 'list': 'array', + 'List': 'array', + 'Union': 'object', + 'Input': 'object', + 'object': 'object', +} + + +def generate_pipeline_parameters_schema(parameters): + parameters_schema = {'type': 'object', 'properties': {}} + if len(parameters) == 0: + return {} + for param in parameters: + name, param_type, has_default, default_value = param + # 'max_length': ('int', True, 1024) + prop = {'type': meta_type_schema_map[param_type]} + if has_default: + prop['default'] = default_value + parameters_schema['properties'][name] = prop + return parameters_schema + + +def get_pipeline_information_by_pipeline(pipeline: Pipeline, ): + """Get pipeline input output schema. + + Args: + pipeline (Pipeline): The pipeline object. + """ + task_name = pipeline.group_key + pipeline_class = pipeline.__class__.__name__ + spec = importlib.util.find_spec(pipeline.__module__) + pipeline_file_path = spec.origin + info = PipelineInfomation(task_name, pipeline_class, pipeline_file_path) + return info + + +class PipelineInfomation(): + """Analyze pipeline information, task_name, schema. + """ + + def __init__(self, task_name: str, class_name, source_path): + self._task_name = task_name + self._class_name = class_name + self._source_path = source_path + self._is_custom_call_method = False + self._analyze() + + def _analyze(self): + input, parameters = get_pipeline_input_parameters( + self._source_path, self._class_name) + if input is not None: # custom pipeline __call__ asr_inferrnce_pipeline + self._is_custom_call_method = True + self._input_schema = generate_pipeline_parameters_schema(input) + self._input_schema[ + 'description'] = 'For binary input such as image audio video, only url is supported.' + self._parameters_schema = {} + self._output_schema = { + 'type': 'object', + } + if self._task_name in TASK_OUTPUTS: + self._output_schema = get_output_schema(self._task_name) + else: + # use base pipeline __call__ + if self._task_name in TASK_INPUTS and self._task_name in TASK_OUTPUTS: + # delete the first default input which is defined by task. + self._parameters_schema = generate_pipeline_parameters_schema( + parameters) + self._input_schema = get_input_schema(self._task_name, None) + self._output_schema = get_output_schema(self._task_name) + else: + logger.warning( + 'Task: %s input is defined: %s, output is defined: %s which is not completed' + % (self._task_name, self._task_name + in TASK_INPUTS, self._task_name in TASK_OUTPUTS)) + self._input_schema = None + self._output_schema = None + if self._task_name in TASK_INPUTS: + self._input_schema = get_input_schema( + self._task_name, None) + if self._task_name in TASK_OUTPUTS: + self._output_schema = get_output_schema(self._task_name) + self._parameters_schema = generate_pipeline_parameters_schema( + parameters) + + @property + def task_name(self): + return self._task_name + + @property + def is_custom_call(self): + return self._is_custom_call_method + + @property + def input_schema(self): + return self._input_schema + + @property + def output_schema(self): + return self._output_schema + + @property + def parameters_schema(self): + return self._parameters_schema + + @property + def schema(self): + return { + 'input': self._input_schema if self._input_schema else + self._parameters_schema, # all parameter is input + 'parameters': + self._parameters_schema if self._input_schema else {}, + 'output': self._output_schema if self._output_schema else { + 'type': 'object', + }, + } + + +def is_url(url: str): + """Check the input url is valid url. + + Args: + url (str): The url + + Returns: + bool: If is url return True, otherwise False. + """ + url_parsed = urlparse(url) + if url_parsed.scheme in ('http', 'https', 'oss'): + return True + else: + return False + + +def decode_base64_to_image(content): + if content.startswith('http') or content.startswith('oss'): + return content + + from PIL import Image + image_file_content = base64.b64decode(content) + return Image.open(BytesIO(image_file_content)) + + +def decode_base64_to_audio(content): + if content.startswith('http') or content.startswith('oss'): + return content + + file_content = base64.b64decode(content) + return file_content + + +def decode_base64_to_video(content): + if content.startswith('http') or content.startswith('oss'): + return content + + file_content = base64.b64decode(content) + return file_content + + +def return_origin(content): + return content + + +def decode_box(content): + pass + + +def service_multipart_input_to_pipeline_input(body): + """Convert multipart data to pipeline input. + + Args: + body (dict): The multipart data body + """ + pass + + +def pipeline_output_to_service_multipart_output(output): + """Convert multipart data to service multipart output. + + Args: + output (dict): Multipart body. + """ + pass + + +base64_decoder_map = { + InputType.IMAGE: decode_base64_to_image, + InputType.TEXT: return_origin, + InputType.AUDIO: decode_base64_to_audio, + InputType.VIDEO: decode_base64_to_video, + InputType.BOX: decode_box, + InputType.DICT: return_origin, + InputType.LIST: return_origin, + InputType.NUMBER: return_origin, +} + + +def call_pipeline_with_json(pipeline_info: PipelineInfomation, + pipeline: Pipeline, body: str): + """Call pipeline with json input. + + Args: + pipeline_info (PipelineInfomation): The pipeline information object. + pipeline (Pipeline): The pipeline object. + body (Dict): The input object, include input and parameters + """ + if pipeline_info.is_custom_call: + pipeline_inputs = body['input'] + result = pipeline(**pipeline_inputs) + else: + pipeline_inputs, parameters = service_base64_input_to_pipeline_input( + pipeline_info.task_name, body) + result = pipeline(pipeline_inputs, **parameters) + + return result + + +def service_base64_input_to_pipeline_input(task_name, body): + """Convert service base64 input to pipeline input and parameters + + Args: + task_name (str): The task name. + body (Dict): The input object, include input and parameters + """ + if 'input' not in body: + raise ValueError('No input data!') + service_input = body['input'] + if 'parameters' in body: + parameters = body['parameters'] + else: + parameters = {} + pipeline_input = {} + + task_input_info = TASK_INPUTS[task_name] + if isinstance(task_input_info, str): # no input key default + return base64_decoder_map[task_input_info](list( + service_input.values())[0]), parameters + elif isinstance(task_input_info, tuple): + pipeline_input = tuple(service_input) + return pipeline_input, parameters + elif isinstance(task_input_info, dict): + for key, value in service_input.items( + ): # task input has no nesting field. + # get input filed type + input_type = task_input_info[key] + # TODO recursion for list, dict if need. + if not isinstance(input_type, str): + pipeline_input[key] = value + continue + if input_type not in INPUT_TYPE: + raise ValueError('Invalid input field: %s' % input_type) + pipeline_input[key] = base64_decoder_map[input_type](value) + return pipeline_input, parameters + elif isinstance(task_input_info, + list): # one of input format, we use dict. + for item in task_input_info: + if isinstance(item, dict): + for key, value in service_input.items( + ): # task input has no nesting field. + # get input filed type + input_type = item[key] + if input_type not in INPUT_TYPE: + raise ValueError('Invalid input field: %s' + % input_type) + pipeline_input[key] = base64_decoder_map[input_type](value) + return pipeline_input, parameters + else: + raise IndexError('Task %s input invalid: %s' % + (task_name, task_input_info)) + + +def encode_numpy_image_to_base64(image): + from PIL import Image + with BytesIO() as output_bytes: + pil_image = Image.fromarray(image.astype(np.uint8)) + pil_image.save(output_bytes, 'PNG') + bytes_data = output_bytes.getvalue() + base64_str = str(base64.b64encode(bytes_data), 'utf-8') + return base64_str + + +def encode_video_to_base64(video): + return str(base64.b64encode(video), 'utf-8') + + +def encode_pcm_to_base64(pcm): + return str(base64.b64encode(pcm), 'utf-8') + + +def encode_wav_to_base64(wav): + return str(base64.b64encode(wav), 'utf-8') + + +def encode_bytes_to_base64(bts): + return str(base64.b64encode(bts), 'utf-8') + + +base64_encoder_map = { + 'image': encode_numpy_image_to_base64, + 'video': encode_video_to_base64, + 'pcm': encode_pcm_to_base64, + 'wav': encode_wav_to_base64, + 'bytes': encode_bytes_to_base64, +} + +# convert numpy etc type to python type. +type_to_python_type = { + np.int64: int, +} + + +def _convert_to_python_type(inputs): + if isinstance(inputs, (list, tuple)): + res = [] + for item in inputs: + res.append(_convert_to_python_type(item)) + return res + elif isinstance(inputs, dict): + res = {} + for k, v in inputs.items(): + if type(v) in type_to_python_type: + res[k] = type_to_python_type[type(v)](v) + else: + res[k] = _convert_to_python_type(v) + return res + else: + return inputs + + +def pipeline_output_to_service_base64_output(task_name, pipeline_output): + """Convert pipeline output to service output, + convert binary fields to base64 encoding。 + + Args: + task_name (str): The output task name. + pipeline_output (object): The pipeline output. + """ + json_serializable_output = {} + task_outputs = [] + if task_name in TASK_OUTPUTS: + task_outputs = TASK_OUTPUTS[task_name] + for key, value in pipeline_output.items(): + if key not in task_outputs: + continue # skip the output not defined. + if key in [ + OutputKeys.OUTPUT_IMG, OutputKeys.OUTPUT_IMGS, + OutputKeys.OUTPUT_VIDEO, OutputKeys.OUTPUT_PCM, + OutputKeys.OUTPUT_PCM_LIST, OutputKeys.OUTPUT_WAV + ]: + if isinstance(value, list): + items = [] + if key == OutputKeys.OUTPUT_IMGS: + output_item_type = OutputKeys.OUTPUT_IMG + else: + output_item_type = OutputKeys.OUTPUT_PCM + for item in value: + items.append(base64_encoder_map[output_item_type](item)) + json_serializable_output[key] = items + else: + json_serializable_output[key] = base64_encoder_map[ + OutputTypes[key]]( + value) + elif OutputTypes[key] in [np.ndarray]: + json_serializable_output[key] = value.tolist() + else: + json_serializable_output[key] = value + + return _convert_to_python_type(json_serializable_output) diff --git a/tests/pipelines/test_table_question_answering.py b/tests/pipelines/test_table_question_answering.py index 4d6eff24..d688ef23 100644 --- a/tests/pipelines/test_table_question_answering.py +++ b/tests/pipelines/test_table_question_answering.py @@ -40,7 +40,7 @@ def tableqa_tracking_and_print_results_with_history( print('question', question) print('sql text:', output_dict[OutputKeys.SQL_STRING]) print('sql query:', output_dict[OutputKeys.SQL_QUERY]) - print('query result:', output_dict[OutputKeys.QUERT_RESULT]) + print('query result:', output_dict[OutputKeys.QUERY_RESULT]) print('json dumps', json.dumps(output_dict, ensure_ascii=False)) print() historical_queries = output_dict[OutputKeys.HISTORY] @@ -66,7 +66,7 @@ def tableqa_tracking_and_print_results_without_history( print('question', question) print('sql text:', output_dict[OutputKeys.SQL_STRING]) print('sql query:', output_dict[OutputKeys.SQL_QUERY]) - print('query result:', output_dict[OutputKeys.QUERT_RESULT]) + print('query result:', output_dict[OutputKeys.QUERY_RESULT]) print('json dumps', json.dumps(output_dict, ensure_ascii=False)) print() @@ -99,7 +99,7 @@ def tableqa_tracking_and_print_results_with_tableid( print('question', question) print('sql text:', output_dict[OutputKeys.SQL_STRING]) print('sql query:', output_dict[OutputKeys.SQL_QUERY]) - print('query result:', output_dict[OutputKeys.QUERT_RESULT]) + print('query result:', output_dict[OutputKeys.QUERY_RESULT]) print('json dumps', json.dumps(output_dict, ensure_ascii=False)) print() historical_queries = output_dict[OutputKeys.HISTORY] @@ -135,7 +135,7 @@ class TableQuestionAnswering(unittest.TestCase): 'history_sql': None }) print(i, result[OutputKeys.OUTPUT][OutputKeys.SQL_QUERY], - result[OutputKeys.OUTPUT][OutputKeys.QUERT_RESULT], + result[OutputKeys.OUTPUT][OutputKeys.QUERY_RESULT], json.dumps(result)) procs = [] diff --git a/tests/utils/test_input_output.py b/tests/utils/test_input_output.py new file mode 100644 index 00000000..53b75a39 --- /dev/null +++ b/tests/utils/test_input_output.py @@ -0,0 +1,142 @@ +import base64 +import unittest + +import json + +from modelscope.utils.constant import Tasks +from modelscope.utils.input_output import ( + PipelineInfomation, service_base64_input_to_pipeline_input) + + +def encode_image_to_base64(image): + base64_str = str(base64.b64encode(image), 'utf-8') + return base64_str + + +class PipelineInputOutputTest(unittest.TestCase): + + def test_template_pipeline_dict_input(self): + pipeline_info = PipelineInfomation( + Tasks.task_template, 'PipelineTemplate', + 'modelscope/pipelines/pipeline_template.py') + schema = pipeline_info.schema + expect_schema = { + 'input': { + 'type': 'object', + 'properties': { + 'image': { + 'type': 'string', + 'description': + 'Base64 encoded image file or url string.' + }, + 'text': { + 'type': 'string', + 'description': 'The input text.' + } + } + }, + 'parameters': { + 'type': 'object', + 'properties': { + 'max_length': { + 'type': 'integer', + 'default': 1024 + }, + 'top_p': { + 'type': 'number', + 'default': 0.8 + }, + 'postprocess_param1': { + 'type': 'string', + 'default': None + } + } + }, + 'output': { + 'type': 'object', + 'properties': { + 'boxes': { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, + 'output_img': { + 'type': 'string', + 'description': 'The base64 encoded image.' + }, + 'text_embedding': { + 'type': 'array', + 'items': { + 'type': 'number' + } + } + } + } + } + assert expect_schema == schema + + def test_template_pipeline_list_input(self): + pipeline_info = PipelineInfomation( + Tasks.text_classification, 'LanguageIdentificationPipeline', + 'modelscope/pipelines/nlp/language_identification_pipline.py') + schema = pipeline_info.schema + expect_schema = { + 'input': { + 'type': 'object', + 'properties': { + 'text': { + 'type': 'string', + 'description': 'The input text.' + }, + 'text2': { + 'type': 'string', + 'description': 'The input text.' + } + } + }, + 'parameters': {}, + 'output': { + 'type': 'object', + 'properties': { + 'scores': { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, + 'labels': { + 'type': 'array', + 'items': { + 'type': 'string' + } + } + } + } + } + assert expect_schema == schema + + def test_input_output_encode_decode(self): + with open('data/test/images/image_captioning.png', 'rb') as f: + image = f.read() + text = 'hello schema.' + request_json = { + 'input': { + 'image': encode_image_to_base64(image), + 'text': text + }, + 'parameters': { + 'max_length': 10000, + 'top_p': 0.8 + } + } + pipeline_inputs, parameters = service_base64_input_to_pipeline_input( + Tasks.task_template, request_json) + assert 'image' in pipeline_inputs + assert pipeline_inputs['text'] == text + assert parameters['max_length'] == 10000 + assert parameters['top_p'] == 0.8 + + +if __name__ == '__main__': + unittest.main() From ca85447363ea2b53405d7f14b13ace2bf224f6a5 Mon Sep 17 00:00:00 2001 From: "hemu.zp" Date: Thu, 11 May 2023 14:31:18 +0800 Subject: [PATCH 20/76] remove pai-easycv requirement Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12564234 --- requirements/cv.txt | 1 - requirements/framework.txt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements/cv.txt b/requirements/cv.txt index 1c9ed435..3230d3fe 100644 --- a/requirements/cv.txt +++ b/requirements/cv.txt @@ -39,7 +39,6 @@ onnxruntime>=1.10 onnxsim open-clip-torch>=2.7.0 opencv-python -pai-easycv>=0.8,<0.10.0 paint_ldm pandas panopticapi diff --git a/requirements/framework.txt b/requirements/framework.txt index 144cd31e..88d60bc1 100644 --- a/requirements/framework.txt +++ b/requirements/framework.txt @@ -16,7 +16,7 @@ python-dateutil>=2.1 pyyaml requests scipy -setuptools==59.8.0 +setuptools simplejson>=3.3.0 sortedcontainers>=1.5.9 tqdm>=4.64.0 From 58df4481822376eec69cf139c49aa4114a72914b Mon Sep 17 00:00:00 2001 From: "wanyu.wy" Date: Thu, 11 May 2023 14:41:08 +0800 Subject: [PATCH 21/76] Add trainer for UniTE --- modelscope/metainfo.py | 3 + modelscope/metrics/__init__.py | 4 +- modelscope/metrics/builder.py | 2 + .../metrics/translation_evaluation_metric.py | 174 ++++++++ modelscope/models/nlp/unite/__init__.py | 8 +- ...onfiguration_unite.py => configuration.py} | 2 +- ...ing_unite.py => translation_evaluation.py} | 256 ++++++----- modelscope/outputs/nlp_outputs.py | 10 + modelscope/outputs/outputs.py | 4 +- .../nlp/translation_evaluation_pipeline.py | 65 ++- modelscope/preprocessors/__init__.py | 8 +- modelscope/preprocessors/nlp/__init__.py | 4 +- .../translation_evaluation_preprocessor.py | 153 +++++-- modelscope/trainers/nlp/__init__.py | 4 +- .../nlp/translation_evaluation_trainer.py | 396 ++++++++++++++++++ .../test_translation_evaluation_metrics.py | 30 ++ .../pipelines/test_translation_evaluation.py | 46 +- tests/run_config.yaml | 2 + tests/trainers/model_trainer_map.py | 6 + .../test_translation_evaluation_trainer.py | 30 ++ 20 files changed, 987 insertions(+), 220 deletions(-) create mode 100644 modelscope/metrics/translation_evaluation_metric.py rename modelscope/models/nlp/unite/{configuration_unite.py => configuration.py} (93%) rename modelscope/models/nlp/unite/{modeling_unite.py => translation_evaluation.py} (61%) create mode 100644 modelscope/trainers/nlp/translation_evaluation_trainer.py create mode 100644 tests/metrics/test_translation_evaluation_metrics.py create mode 100644 tests/trainers/test_translation_evaluation_trainer.py diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index 0d7b5f64..bc3d99b9 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -875,6 +875,7 @@ class NLPTrainers(object): document_grounded_dialog_rerank_trainer = 'document-grounded-dialog-rerank-trainer' document_grounded_dialog_retrieval_trainer = 'document-grounded-dialog-retrieval-trainer' siamese_uie_trainer = 'siamese-uie-trainer' + translation_evaluation_trainer = 'translation-evaluation-trainer' class MultiModalTrainers(object): @@ -1089,6 +1090,8 @@ class Metrics(object): # metric for image-colorization task image_colorization_metric = 'image-colorization-metric' ocr_recognition_metric = 'ocr-recognition-metric' + # metric for translation evaluation + translation_evaluation_metric = 'translation-evaluation-metric' class Optimizers(object): diff --git a/modelscope/metrics/__init__.py b/modelscope/metrics/__init__.py index 17767001..6f5dfbde 100644 --- a/modelscope/metrics/__init__.py +++ b/modelscope/metrics/__init__.py @@ -31,6 +31,7 @@ if TYPE_CHECKING: from .loss_metric import LossMetric from .image_colorization_metric import ImageColorizationMetric from .ocr_recognition_metric import OCRRecognitionMetric + from .translation_evaluation_metric import TranslationEvaluationMetric else: _import_structure = { 'audio_noise_metric': ['AudioNoiseMetric'], @@ -62,7 +63,8 @@ else: 'text_ranking_metric': ['TextRankingMetric'], 'loss_metric': ['LossMetric'], 'image_colorization_metric': ['ImageColorizationMetric'], - 'ocr_recognition_metric': ['OCRRecognitionMetric'] + 'ocr_recognition_metric': ['OCRRecognitionMetric'], + 'translation_evaluation_metric': ['TranslationEvaluationMetric'] } import sys diff --git a/modelscope/metrics/builder.py b/modelscope/metrics/builder.py index 2bc756e6..43aaea14 100644 --- a/modelscope/metrics/builder.py +++ b/modelscope/metrics/builder.py @@ -42,6 +42,7 @@ class MetricKeys(object): NDCG = 'ndcg' AR = 'AR' Colorfulness = 'colorfulness' + Kendall_Tau_Correlation = 'kendall_tau_correlation' task_default_metrics = { @@ -76,6 +77,7 @@ task_default_metrics = { Tasks.bad_image_detecting: [Metrics.accuracy], Tasks.ocr_recognition: [Metrics.ocr_recognition_metric], Tasks.efficient_diffusion_tuning: [Metrics.loss_metric], + Tasks.translation_evaluation: [Metrics.translation_evaluation_metric] } diff --git a/modelscope/metrics/translation_evaluation_metric.py b/modelscope/metrics/translation_evaluation_metric.py new file mode 100644 index 00000000..81705d3b --- /dev/null +++ b/modelscope/metrics/translation_evaluation_metric.py @@ -0,0 +1,174 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import importlib +from typing import Dict, List, Union + +from pandas import DataFrame + +from modelscope.metainfo import Metrics +from modelscope.metrics.base import Metric +from modelscope.metrics.builder import METRICS, MetricKeys +from modelscope.models.nlp.unite.configuration import InputFormat +from modelscope.utils.logger import get_logger +from modelscope.utils.registry import default_group + +logger = get_logger() + + +@METRICS.register_module( + group_key=default_group, module_name=Metrics.translation_evaluation_metric) +class TranslationEvaluationMetric(Metric): + r"""The metric class for translation evaluation. + + """ + + def __init__(self, gap_threshold: float = 25.0): + r"""Build a translation evaluation metric, following the designed + Kendall's tau correlation from WMT Metrics Shared Task competitions. + + Args: + gap_threshold: The score gap denoting the available hypothesis pair. + + Returns: + A metric for translation evaluation. + """ + self.gap_threshold = gap_threshold + + self.lp = list() + self.segment_id = list() + self.raw_score = list() + self.score = list() + self.input_format = list() + + def clear(self) -> None: + r"""Clear all the stored variables. + """ + self.lp.clear() + self.segment_id.clear() + self.raw_score.clear() + self.input_format.clear() + + self.score.clear() + + return + + def add(self, outputs: Dict[str, List[float]], + inputs: Dict[str, List[Union[float, int]]]) -> None: + r"""Collect the related results for processing. + + Args: + outputs: Dict containing 'scores' + inputs: Dict containing 'labels' and 'segment_ids' + + """ + + self.lp += inputs['lp'] + self.segment_id += inputs['segment_id'] + self.raw_score += inputs['raw_score'] + self.input_format += inputs['input_format'] + + self.score += outputs['score'] + + return + + def evaluate(self) -> Dict[str, Dict[str, float]]: + r"""Compute the Kendall's tau correlation. + + Returns: + A dict denoting Kendall's tau correlation. + + """ + + data = { + 'lp': self.lp, + 'segment_id': self.segment_id, + 'raw_score': self.raw_score, + 'input_format': self.input_format, + 'score': self.score + } + data = DataFrame(data=data) + correlation = dict() + + for input_format in data.input_format.unique(): + logger.info('Evaluation results for %s input format' + % input_format.value) + input_format_data = data[data.input_format == input_format] + + temp_correlation = dict() + + for lp in sorted(input_format_data.lp.unique()): + sub_data = input_format_data[input_format_data.lp == lp] + temp_correlation[input_format.value + '_' + + lp] = self.compute_kendall_tau(sub_data) + logger.info( + '\t%s: %f' % + (lp, + temp_correlation[input_format.value + '_' + lp] * 100)) + + avg_correlation = sum( + temp_correlation.values()) / len(temp_correlation) + correlation[input_format.value + '_avg'] = avg_correlation + logger.info('Average evaluation result for %s input format: %f' % + (input_format.value, avg_correlation)) + logger.info('') + correlation.update(temp_correlation) + + return correlation + + def merge(self, other: 'TranslationEvaluationMetric') -> None: + r"""Merge the predictions from other TranslationEvaluationMetric objects. + + Args: + other: Another TranslationEvaluationMetric object. + + """ + + self.lp += other.lp + self.segment_id += other.segment_ids + self.raw_score += other.raw_score + self.input_format += other.input_format + + self.score += other.score + + return + + def compute_kendall_tau(self, csv_data: DataFrame) -> float: + r"""Compute kendall's tau correlation. + + Args: + csv_data: The pandas dataframe. + + Returns: + float: THe kendall's Tau correlation. + + """ + concor = discor = 0 + + for segment_id in sorted(csv_data.segment_id.unique()): + group_csv_data = csv_data[csv_data.segment_id == segment_id] + + examples = group_csv_data.to_dict('records') + + for i in range(0, len(examples)): + for j in range(i + 1, len(examples)): + if self.raw_score[i] - self.raw_score[ + j] >= self.gap_threshold: + if self.score[i] > self.score[j]: + concor += 1 + elif self.score[i] < self.score[j]: + discor += 1 + elif self.raw_score[i] - self.raw_score[ + j] <= -self.gap_threshold: + if self.score[i] < self.score[j]: + concor += 1 + elif self.score[i] > self.score[j]: + discor += 1 + + if concor + discor == 0: + logger.warning( + 'We don\'t have available pairs when evaluation. ' + 'Marking the kendall tau correlation as the lowest value (-1.0).' + ) + return -1.0 + else: + return (concor - discor) / (concor + discor) diff --git a/modelscope/models/nlp/unite/__init__.py b/modelscope/models/nlp/unite/__init__.py index 06c2146e..939f0ab7 100644 --- a/modelscope/models/nlp/unite/__init__.py +++ b/modelscope/models/nlp/unite/__init__.py @@ -5,12 +5,12 @@ from typing import TYPE_CHECKING from modelscope.utils.import_utils import LazyImportModule if TYPE_CHECKING: - from .configuration_unite import UniTEConfig - from .modeling_unite import UniTEForTranslationEvaluation + from .configuration import UniTEConfig + from .translation_evaluation import UniTEForTranslationEvaluation else: _import_structure = { - 'configuration_unite': ['UniTEConfig'], - 'modeling_unite': ['UniTEForTranslationEvaluation'], + 'configuration': ['UniTEConfig'], + 'translation_evaluation': ['UniTEForTranslationEvaluation'], } import sys diff --git a/modelscope/models/nlp/unite/configuration_unite.py b/modelscope/models/nlp/unite/configuration.py similarity index 93% rename from modelscope/models/nlp/unite/configuration_unite.py rename to modelscope/models/nlp/unite/configuration.py index b0a48585..402538f7 100644 --- a/modelscope/models/nlp/unite/configuration_unite.py +++ b/modelscope/models/nlp/unite/configuration.py @@ -9,7 +9,7 @@ from modelscope.utils.config import Config logger = logging.get_logger() -class EvaluationMode(Enum): +class InputFormat(Enum): SRC = 'src' REF = 'ref' SRC_REF = 'src-ref' diff --git a/modelscope/models/nlp/unite/modeling_unite.py b/modelscope/models/nlp/unite/translation_evaluation.py similarity index 61% rename from modelscope/models/nlp/unite/modeling_unite.py rename to modelscope/models/nlp/unite/translation_evaluation.py index deea737d..c7e96027 100644 --- a/modelscope/models/nlp/unite/modeling_unite.py +++ b/modelscope/models/nlp/unite/translation_evaluation.py @@ -20,6 +20,8 @@ from transformers.activations import ACT2FN from modelscope.metainfo import Models from modelscope.models.base import TorchModel from modelscope.models.builder import MODELS +from modelscope.models.nlp.unite.configuration import InputFormat +from modelscope.outputs.nlp_outputs import TranslationEvaluationOutput from modelscope.utils.constant import Tasks from modelscope.utils.logger import get_logger @@ -71,8 +73,16 @@ class LayerwiseAttention(Module): mask: torch.Tensor = None, ) -> torch.Tensor: tensors = torch.cat(list(x.unsqueeze(dim=0) for x in tensors), dim=0) - normed_weights = softmax( - self.scalar_parameters, dim=0).view(-1, 1, 1, 1) + + if self.training and self.dropout: + normed_weights = softmax( + torch.where(self.dropout_mask.uniform_() > self.dropout, + self.scalar_parameters, self.dropout_fill), + dim=-1) + else: + normed_weights = softmax(self.scalar_parameters, dim=-1) + + normed_weights = normed_weights.view(-1, 1, 1, 1) mask_float = mask.float() weighted_sum = (normed_weights @@ -97,18 +107,18 @@ class FeedForward(Module): Feed Forward Neural Network. Args: - in_dim (:obj:`int`): - Number of input features. - out_dim (:obj:`int`, defaults to 1): - Number of output features. Default is 1 -- a single scalar. - hidden_sizes (:obj:`List[int]`, defaults to `[3072, 768]`): - List with hidden layer sizes. - activations (:obj:`str`, defaults to `Sigmoid`): - Name of the activation function to be used in the hidden layers. - final_activation (:obj:`str`, Optional, defaults to `None`): - Name of the final activation function if any. - dropout (:obj:`float`, defaults to 0.1): - Dropout ratio to be used in the hidden layers. + in_dim (:obj:`int`): + Number of input features. + out_dim (:obj:`int`, defaults to 1): + Number of output features. Default is 1 -- a single scalar. + hidden_sizes (:obj:`List[int]`, defaults to `[3072, 768]`): + List with hidden layer sizes. + activations (:obj:`str`, defaults to `Sigmoid`): + Name of the activation function to be used in the hidden layers. + final_activation (:obj:`str`, Optional, defaults to `None`): + Name of the final activation function if any. + dropout (:obj:`float`, defaults to 0.1): + Dropout ratio to be used in the hidden layers. """ super().__init__() modules = [] @@ -266,8 +276,11 @@ class UniTEForTranslationEvaluation(TorchModel): return - def forward(self, input_sentences: List[torch.Tensor]): - input_ids = self.combine_input_sentences(input_sentences) + def forward(self, + input_ids: torch.Tensor, + input_format: Optional[List[InputFormat]] = None, + score: Optional[torch.Tensor] = None, + **kwargs) -> TranslationEvaluationOutput: attention_mask = input_ids.ne(self.pad_token_id).long() outputs = self.encoder( input_ids=input_ids, @@ -276,125 +289,138 @@ class UniTEForTranslationEvaluation(TorchModel): return_dict=True) mix_states = self.layerwise_attention(outputs['hidden_states'], attention_mask) - pred = self.estimator(mix_states) - return pred.squeeze(dim=-1) + pred = self.estimator(mix_states).squeeze(dim=-1) + output = TranslationEvaluationOutput( + score=pred.cpu().tolist(), input_format=input_format) - def load_checkpoint(self, path: str, device: torch.device): - state_dict = torch.load(path, map_location=device) - self.load_state_dict(state_dict) + if score is not None: + loss = (pred - score).pow(2).mean() + output['loss'] = loss + + return output + + def load_checkpoint(self, path: str, device: torch.device, plm_only: bool): + if plm_only: + self.encoder = self.encoder.from_pretrained(path).to(device) + self.encoder.pooler = None + else: + state_dict = torch.load(path, map_location=device) + self.load_state_dict(state_dict) logger.info('Loading checkpoint parameters from %s' % path) return - def combine_input_sentences(self, input_sent_groups: List[torch.Tensor]): - for input_sent_group in input_sent_groups[1:]: - input_sent_group[:, 0] = self.eos_token_id - if len(input_sent_groups) == 3: - cutted_sents = self.cut_long_sequences3(input_sent_groups) - else: - cutted_sents = self.cut_long_sequences2(input_sent_groups) - return cutted_sents - - @staticmethod - def cut_long_sequences2(all_input_concat: List[List[torch.Tensor]], +def combine_input_sentences(all_input_concat: List[List[torch.Tensor]], maximum_length: int = 512, - pad_idx: int = 1): - all_input_concat = list(zip(*all_input_concat)) - collected_tuples = list() - for tensor_tuple in all_input_concat: - all_lens = tuple(len(x) for x in tensor_tuple) + pad_idx: int = 1, + eos_idx: int = 2): + for group in all_input_concat[1:]: + group[:, 0] = eos_idx - if sum(all_lens) > maximum_length: - lengths = dict(enumerate(all_lens)) - lengths_sorted_idxes = list(x[0] for x in sorted( - lengths.items(), key=lambda d: d[1], reverse=True)) + if len(all_input_concat) == 3: + return cut_long_sequences3(all_input_concat, maximum_length, pad_idx) + else: + return cut_long_sequences2(all_input_concat, maximum_length, pad_idx) - offset = ceil((sum(lengths.values()) - maximum_length) / 2) - if min(all_lens) > (maximum_length - // 2) and min(all_lens) > offset: - lengths = dict((k, v - offset) for k, v in lengths.items()) - else: - lengths[lengths_sorted_idxes[ - 0]] = maximum_length - lengths[lengths_sorted_idxes[1]] +def cut_long_sequences2(all_input_concat: List[List[torch.Tensor]], + maximum_length: int = 512, + pad_idx: int = 1): + all_input_concat = list(zip(*all_input_concat)) + collected_tuples = list() + for tensor_tuple in all_input_concat: + tensor_tuple = tuple( + x.masked_select(x.ne(pad_idx)) for x in tensor_tuple) + all_lens = tuple(len(x) for x in tensor_tuple) - new_lens = list(lengths[k] - for k in range(0, len(tensor_tuple))) - new_tensor_tuple = tuple( - x[:y] for x, y in zip(tensor_tuple, new_lens)) - for x, y in zip(new_tensor_tuple, tensor_tuple): - x[-1] = y[-1] - collected_tuples.append(new_tensor_tuple) + if sum(all_lens) > maximum_length: + lengths = dict(enumerate(all_lens)) + lengths_sorted_idxes = list(x[0] for x in sorted( + lengths.items(), key=lambda d: d[1], reverse=True)) + + offset = ceil((sum(lengths.values()) - maximum_length) / 2) + + if min(all_lens) > (maximum_length + // 2) and min(all_lens) > offset: + lengths = dict((k, v - offset) for k, v in lengths.items()) else: - collected_tuples.append(tensor_tuple) + lengths[lengths_sorted_idxes[0]] = maximum_length - lengths[ + lengths_sorted_idxes[1]] - concat_tensor = list(torch.cat(x, dim=0) for x in collected_tuples) - all_input_concat_padded = pad_sequence( - concat_tensor, batch_first=True, padding_value=pad_idx) + new_lens = list(lengths[k] for k in range(0, len(tensor_tuple))) + new_tensor_tuple = tuple(x[:y] + for x, y in zip(tensor_tuple, new_lens)) + for x, y in zip(new_tensor_tuple, tensor_tuple): + x[-1] = y[-1] + collected_tuples.append(new_tensor_tuple) + else: + collected_tuples.append(tensor_tuple) - return all_input_concat_padded + concat_tensor = list(torch.cat(x, dim=0) for x in collected_tuples) + all_input_concat_padded = pad_sequence( + concat_tensor, batch_first=True, padding_value=pad_idx) + return all_input_concat_padded - @staticmethod - def cut_long_sequences3(all_input_concat: List[List[torch.Tensor]], - maximum_length: int = 512, - pad_idx: int = 1): - all_input_concat = list(zip(*all_input_concat)) - collected_tuples = list() - for tensor_tuple in all_input_concat: - all_lens = tuple(len(x) for x in tensor_tuple) - if sum(all_lens) > maximum_length: - lengths = dict(enumerate(all_lens)) - lengths_sorted_idxes = list(x[0] for x in sorted( - lengths.items(), key=lambda d: d[1], reverse=True)) +def cut_long_sequences3(all_input_concat: List[List[torch.Tensor]], + maximum_length: int = 512, + pad_idx: int = 1): + all_input_concat = list(zip(*all_input_concat)) + collected_tuples = list() + for tensor_tuple in all_input_concat: + tensor_tuple = tuple( + x.masked_select(x.ne(pad_idx)) for x in tensor_tuple) + all_lens = tuple(len(x) for x in tensor_tuple) - offset = ceil((sum(lengths.values()) - maximum_length) / 3) + if sum(all_lens) > maximum_length: + lengths = dict(enumerate(all_lens)) + lengths_sorted_idxes = list(x[0] for x in sorted( + lengths.items(), key=lambda d: d[1], reverse=True)) - if min(all_lens) > (maximum_length - // 3) and min(all_lens) > offset: - lengths = dict((k, v - offset) for k, v in lengths.items()) - else: - while sum(lengths.values()) > maximum_length: - if lengths[lengths_sorted_idxes[0]] > lengths[ - lengths_sorted_idxes[1]]: - offset = maximum_length - lengths[ - lengths_sorted_idxes[1]] - lengths[ - lengths_sorted_idxes[2]] - if offset > lengths[lengths_sorted_idxes[1]]: - lengths[lengths_sorted_idxes[0]] = offset - else: - lengths[lengths_sorted_idxes[0]] = lengths[ - lengths_sorted_idxes[1]] - elif lengths[lengths_sorted_idxes[0]] == lengths[ - lengths_sorted_idxes[1]] > lengths[ - lengths_sorted_idxes[2]]: - offset = (maximum_length - - lengths[lengths_sorted_idxes[2]]) // 2 - if offset > lengths[lengths_sorted_idxes[2]]: - lengths[lengths_sorted_idxes[0]] = lengths[ - lengths_sorted_idxes[1]] = offset - else: - lengths[lengths_sorted_idxes[0]] = lengths[ - lengths_sorted_idxes[1]] = lengths[ - lengths_sorted_idxes[2]] + offset = ceil((sum(lengths.values()) - maximum_length) / 3) + + if min(all_lens) > (maximum_length + // 3) and min(all_lens) > offset: + lengths = dict((k, v - offset) for k, v in lengths.items()) + else: + while sum(lengths.values()) > maximum_length: + if lengths[lengths_sorted_idxes[0]] > lengths[ + lengths_sorted_idxes[1]]: + offset = maximum_length - lengths[lengths_sorted_idxes[ + 1]] - lengths[lengths_sorted_idxes[2]] + if offset > lengths[lengths_sorted_idxes[1]]: + lengths[lengths_sorted_idxes[0]] = offset + else: + lengths[lengths_sorted_idxes[0]] = lengths[ + lengths_sorted_idxes[1]] + elif lengths[lengths_sorted_idxes[0]] == lengths[ + lengths_sorted_idxes[1]] > lengths[ + lengths_sorted_idxes[2]]: + offset = (maximum_length + - lengths[lengths_sorted_idxes[2]]) // 2 + if offset > lengths[lengths_sorted_idxes[2]]: + lengths[lengths_sorted_idxes[0]] = lengths[ + lengths_sorted_idxes[1]] = offset else: lengths[lengths_sorted_idxes[0]] = lengths[ lengths_sorted_idxes[1]] = lengths[ - lengths_sorted_idxes[ - 2]] = maximum_length // 3 + lengths_sorted_idxes[2]] + else: + lengths[lengths_sorted_idxes[0]] = lengths[ + lengths_sorted_idxes[1]] = lengths[ + lengths_sorted_idxes[2]] = maximum_length // 3 - new_lens = list(lengths[k] for k in range(0, len(lengths))) - new_tensor_tuple = tuple( - x[:y] for x, y in zip(tensor_tuple, new_lens)) + new_lens = list(lengths[k] for k in range(0, len(lengths))) + new_tensor_tuple = tuple(x[:y] + for x, y in zip(tensor_tuple, new_lens)) - for x, y in zip(new_tensor_tuple, tensor_tuple): - x[-1] = y[-1] - collected_tuples.append(new_tensor_tuple) - else: - collected_tuples.append(tensor_tuple) + for x, y in zip(new_tensor_tuple, tensor_tuple): + x[-1] = y[-1] + collected_tuples.append(new_tensor_tuple) + else: + collected_tuples.append(tensor_tuple) - concat_tensor = list(torch.cat(x, dim=0) for x in collected_tuples) - all_input_concat_padded = pad_sequence( - concat_tensor, batch_first=True, padding_value=pad_idx) - - return all_input_concat_padded + concat_tensor = list(torch.cat(x, dim=0) for x in collected_tuples) + all_input_concat_padded = pad_sequence( + concat_tensor, batch_first=True, padding_value=pad_idx) + return all_input_concat_padded diff --git a/modelscope/outputs/nlp_outputs.py b/modelscope/outputs/nlp_outputs.py index e288df70..d6b934c2 100644 --- a/modelscope/outputs/nlp_outputs.py +++ b/modelscope/outputs/nlp_outputs.py @@ -454,3 +454,13 @@ class SentencEmbeddingModelOutput(ModelOutputBase): query_embeddings: Tensor = None doc_embeddings: Tensor = None loss: Tensor = None + + +@dataclass +class TranslationEvaluationOutput(ModelOutputBase): + """The output class for translation evaluation models. + """ + + score: Tensor = None + loss: Tensor = None + input_format: List[str] = None diff --git a/modelscope/outputs/outputs.py b/modelscope/outputs/outputs.py index 1cf891c7..1b06795a 100644 --- a/modelscope/outputs/outputs.py +++ b/modelscope/outputs/outputs.py @@ -1447,9 +1447,9 @@ TASK_OUTPUTS = { # } Tasks.image_skychange: [OutputKeys.OUTPUT_IMG], # { - # 'scores': [0.1, 0.2, 0.3, ...] + # 'score': [0.1, 0.2, 0.3, ...] # } - Tasks.translation_evaluation: [OutputKeys.SCORES], + Tasks.translation_evaluation: [OutputKeys.SCORE], # video object segmentation result for a single video # { diff --git a/modelscope/pipelines/nlp/translation_evaluation_pipeline.py b/modelscope/pipelines/nlp/translation_evaluation_pipeline.py index 1f8ba79a..66d3460c 100644 --- a/modelscope/pipelines/nlp/translation_evaluation_pipeline.py +++ b/modelscope/pipelines/nlp/translation_evaluation_pipeline.py @@ -9,12 +9,11 @@ import torch from modelscope.metainfo import Pipelines from modelscope.models.base import Model -from modelscope.models.nlp.unite.configuration_unite import EvaluationMode +from modelscope.models.nlp.unite.configuration import InputFormat from modelscope.outputs import OutputKeys from modelscope.pipelines.base import InputModel, Pipeline from modelscope.pipelines.builder import PIPELINES -from modelscope.preprocessors import (Preprocessor, - TranslationEvaluationPreprocessor) +from modelscope.preprocessors import Preprocessor from modelscope.utils.config import Config from modelscope.utils.constant import ModelFile, Tasks from modelscope.utils.logger import get_logger @@ -31,57 +30,55 @@ class TranslationEvaluationPipeline(Pipeline): def __init__(self, model: InputModel, preprocessor: Optional[Preprocessor] = None, - eval_mode: EvaluationMode = EvaluationMode.SRC_REF, + input_format: InputFormat = InputFormat.SRC_REF, device: str = 'gpu', **kwargs): - r"""Build a translation pipeline with a model dir or a model id in the model hub. + r"""Build a translation evaluation pipeline with a model dir or a model id in the model hub. Args: model: A Model instance. - eval_mode: Evaluation mode, choosing one from `"EvaluationMode.SRC_REF"`, - `"EvaluationMode.SRC"`, `"EvaluationMode.REF"`. Aside from hypothesis, the + preprocessor: The preprocessor for this pipeline. + input_format: Input format, choosing one from `"InputFormat.SRC_REF"`, + `"InputFormat.SRC"`, `"InputFormat.REF"`. Aside from hypothesis, the source/reference/source+reference can be presented during evaluation. + device: Used device for this pipeline. """ super().__init__(model=model, preprocessor=preprocessor) - self.eval_mode = eval_mode - self.checking_eval_mode() + self.input_format = input_format + self.checking_input_format() assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' - self.preprocessor = TranslationEvaluationPreprocessor( - self.model.model_dir, - self.eval_mode) if preprocessor is None else preprocessor - self.model.load_checkpoint( osp.join(self.model.model_dir, ModelFile.TORCH_MODEL_BIN_FILE), - self.device) + device=self.device, + plm_only=False) self.model.eval() return - def checking_eval_mode(self): - if self.eval_mode == EvaluationMode.SRC: + def checking_input_format(self): + if self.input_format == InputFormat.SRC: logger.info('Evaluation mode: source-only') - elif self.eval_mode == EvaluationMode.REF: + elif self.input_format == InputFormat.REF: logger.info('Evaluation mode: reference-only') - elif self.eval_mode == EvaluationMode.SRC_REF: + elif self.input_format == InputFormat.SRC_REF: logger.info('Evaluation mode: source-reference-combined') else: - raise ValueError( - 'Evaluation mode should be one choice among' - '\'EvaluationMode.SRC\', \'EvaluationMode.REF\', and' - '\'EvaluationMode.SRC_REF\'.') + raise ValueError('Evaluation mode should be one choice among' + '\'InputFormat.SRC\', \'InputFormat.REF\', and' + '\'InputFormat.SRC_REF\'.') - def change_eval_mode(self, - eval_mode: EvaluationMode = EvaluationMode.SRC_REF): + def change_input_format(self, + input_format: InputFormat = InputFormat.SRC_REF): logger.info('Changing the evaluation mode.') - self.eval_mode = eval_mode - self.checking_eval_mode() - self.preprocessor.eval_mode = eval_mode + self.input_format = input_format + self.checking_input_format() + self.preprocessor.change_input_format(input_format) return - def __call__(self, input: Dict[str, Union[str, List[str]]], **kwargs): + def __call__(self, input_dict: Dict[str, Union[str, List[str]]], **kwargs): r"""Implementation of __call__ function. Args: @@ -104,12 +101,12 @@ class TranslationEvaluationPipeline(Pipeline): } ``` """ - return super().__call__(input=input, **kwargs) + return super().__call__(input=input_dict, **kwargs) - def forward(self, - input_ids: List[torch.Tensor]) -> Dict[str, torch.Tensor]: - return self.model(input_ids) + def forward( + self, input_dict: Dict[str, + torch.Tensor]) -> Dict[str, torch.Tensor]: + return self.model(**input_dict) def postprocess(self, output: torch.Tensor) -> Dict[str, Any]: - result = {OutputKeys.SCORES: output.cpu().tolist()} - return result + return output diff --git a/modelscope/preprocessors/__init__.py b/modelscope/preprocessors/__init__.py index a35f130a..8d74f521 100644 --- a/modelscope/preprocessors/__init__.py +++ b/modelscope/preprocessors/__init__.py @@ -41,9 +41,9 @@ if TYPE_CHECKING: DialogStateTrackingPreprocessor, ConversationalTextToSqlPreprocessor, TableQuestionAnsweringPreprocessor, NERPreprocessorViet, NERPreprocessorThai, WordSegmentationPreprocessorThai, - TranslationEvaluationPreprocessor, CanmtTranslationPreprocessor, - DialogueClassificationUsePreprocessor, SiameseUiePreprocessor, - DocumentGroundedDialogGeneratePreprocessor, + TranslationEvaluationTransformersPreprocessor, + CanmtTranslationPreprocessor, DialogueClassificationUsePreprocessor, + SiameseUiePreprocessor, DocumentGroundedDialogGeneratePreprocessor, DocumentGroundedDialogRetrievalPreprocessor, DocumentGroundedDialogRerankPreprocessor) from .video import ReadVideoData, MovieSceneSegmentationPreprocessor @@ -96,7 +96,7 @@ else: 'DialogStateTrackingPreprocessor', 'ConversationalTextToSqlPreprocessor', 'TableQuestionAnsweringPreprocessor', - 'TranslationEvaluationPreprocessor', + 'TranslationEvaluationTransformersPreprocessor', 'CanmtTranslationPreprocessor', 'DialogueClassificationUsePreprocessor', 'SiameseUiePreprocessor', 'DialogueClassificationUsePreprocessor', diff --git a/modelscope/preprocessors/nlp/__init__.py b/modelscope/preprocessors/nlp/__init__.py index 5904d65e..19421fa0 100644 --- a/modelscope/preprocessors/nlp/__init__.py +++ b/modelscope/preprocessors/nlp/__init__.py @@ -29,7 +29,7 @@ if TYPE_CHECKING: from .space_T_en import ConversationalTextToSqlPreprocessor from .space_T_cn import TableQuestionAnsweringPreprocessor from .mglm_summarization_preprocessor import MGLMSummarizationPreprocessor - from .translation_evaluation_preprocessor import TranslationEvaluationPreprocessor + from .translation_evaluation_preprocessor import TranslationEvaluationTransformersPreprocessor from .canmt_translation import CanmtTranslationPreprocessor from .dialog_classification_use_preprocessor import DialogueClassificationUsePreprocessor from .siamese_uie_preprocessor import SiameseUiePreprocessor @@ -90,7 +90,7 @@ else: 'space_T_en': ['ConversationalTextToSqlPreprocessor'], 'space_T_cn': ['TableQuestionAnsweringPreprocessor'], 'translation_evaluation_preprocessor': - ['TranslationEvaluationPreprocessor'], + ['TranslationEvaluationTransformersPreprocessor'], 'canmt_translation': [ 'CanmtTranslationPreprocessor', ], diff --git a/modelscope/preprocessors/nlp/translation_evaluation_preprocessor.py b/modelscope/preprocessors/nlp/translation_evaluation_preprocessor.py index 0bf62cdc..b0b2efd1 100644 --- a/modelscope/preprocessors/nlp/translation_evaluation_preprocessor.py +++ b/modelscope/preprocessors/nlp/translation_evaluation_preprocessor.py @@ -2,10 +2,13 @@ from typing import Any, Dict, List, Union +import torch from transformers import AutoTokenizer from modelscope.metainfo import Preprocessors -from modelscope.models.nlp.unite.configuration_unite import EvaluationMode +from modelscope.models.nlp.unite.configuration import InputFormat +from modelscope.models.nlp.unite.translation_evaluation import \ + combine_input_sentences from modelscope.preprocessors import Preprocessor from modelscope.preprocessors.builder import PREPROCESSORS from modelscope.utils.constant import Fields, ModeKeys @@ -14,43 +17,98 @@ from .transformers_tokenizer import NLPTokenizer @PREPROCESSORS.register_module( Fields.nlp, module_name=Preprocessors.translation_evaluation) -class TranslationEvaluationPreprocessor(Preprocessor): +class TranslationEvaluationTransformersPreprocessor(Preprocessor): r"""The tokenizer preprocessor used for translation evaluation. """ def __init__(self, model_dir: str, - eval_mode: EvaluationMode, + max_len: int, + pad_token_id: int, + eos_token_id: int, + input_format: InputFormat = InputFormat.SRC_REF, mode=ModeKeys.INFERENCE, *args, **kwargs): - r"""preprocess the data via the vocab file from the `model_dir` path + r"""Preprocessing the data for the model in `model_dir` path Args: model_dir: A Model instance. - eval_mode: Evaluation mode, choosing one from `"EvaluationMode.SRC_REF"`, - `"EvaluationMode.SRC"`, `"EvaluationMode.REF"`. Aside from hypothesis, the + max_len: Maximum length for input sequence. + pad_token_id: Token id for padding token. + eos_token_id: Token id for the ending-of-sequence (eos) token. + input_format: Input format, choosing one from `"InputFormat.SRC_REF"`, + `"InputFormat.SRC"`, `"InputFormat.REF"`. Aside from hypothesis, the source/reference/source+reference can be presented during evaluation. + mode: The mode for this preprocessor. """ super().__init__(mode=mode) self.tokenizer = NLPTokenizer( model_dir=model_dir, use_fast=False, tokenize_kwargs=kwargs) - self.eval_mode = eval_mode + self.input_format = input_format + + self.max_len = max_len + self.pad_token_id = pad_token_id + self.eos_token_id = eos_token_id return - def __call__(self, input_dict: Dict[str, Any]) -> List[List[str]]: - if self.eval_mode == EvaluationMode.SRC and 'src' not in input_dict.keys( + def change_input_format(self, input_format: InputFormat): + r"""Change the input format for the preprocessor. + + Args: + input_format: Any choice in InputFormat.SRC_REF, InputFormat.SRC and InputFormat.REF. + + """ + self.input_format = input_format + return + + def collect_input_ids(self, input_dict: Dict[str, Any]): + r"""Collect the input ids for the given examples. + + Args: + input_dict: A dict containing hyp/src/ref sentences. + + Returns: + The token ids for each example. + + """ + output_sents = [ + self.tokenizer( + input_dict['hyp'], return_tensors='pt', + padding=True)['input_ids'] + ] + if self.input_format == InputFormat.SRC or self.input_format == InputFormat.SRC_REF: + output_sents += [ + self.tokenizer( + input_dict['src'], return_tensors='pt', + padding=True)['input_ids'] + ] + if self.input_format == InputFormat.REF or self.input_format == InputFormat.SRC_REF: + output_sents += [ + self.tokenizer( + input_dict['ref'], return_tensors='pt', + padding=True)['input_ids'] + ] + + input_ids = combine_input_sentences(output_sents, self.max_len, + self.pad_token_id, + self.eos_token_id) + + return input_ids + + def __call__(self, input_dict: Dict[str, Any]) -> Dict[str, Any]: + if self.input_format == InputFormat.SRC and 'src' not in input_dict.keys( ): raise ValueError( 'Source sentences are required for source-only evaluation mode.' ) - if self.eval_mode == EvaluationMode.REF and 'ref' not in input_dict.keys( + if self.input_format == InputFormat.REF and 'ref' not in input_dict.keys( ): raise ValueError( 'Reference sentences are required for reference-only evaluation mode.' ) - if self.eval_mode == EvaluationMode.SRC_REF and ( + if self.input_format == InputFormat.SRC_REF and ( 'src' not in input_dict.keys() or 'ref' not in input_dict.keys()): raise ValueError( @@ -59,29 +117,58 @@ class TranslationEvaluationPreprocessor(Preprocessor): if type(input_dict['hyp']) == str: input_dict['hyp'] = [input_dict['hyp']] - if (self.eval_mode == EvaluationMode.SRC or self.eval_mode - == EvaluationMode.SRC_REF) and type(input_dict['src']) == str: + if (self.input_format == InputFormat.SRC or self.input_format + == InputFormat.SRC_REF) and type(input_dict['src']) == str: input_dict['src'] = [input_dict['src']] - if (self.eval_mode == EvaluationMode.REF or self.eval_mode - == EvaluationMode.SRC_REF) and type(input_dict['ref']) == str: + if (self.input_format == InputFormat.REF or self.input_format + == InputFormat.SRC_REF) and type(input_dict['ref']) == str: input_dict['ref'] = [input_dict['ref']] - output_sents = [ - self.tokenizer( - input_dict['hyp'], return_tensors='pt', - padding=True)['input_ids'] - ] - if self.eval_mode == EvaluationMode.SRC or self.eval_mode == EvaluationMode.SRC_REF: - output_sents += [ - self.tokenizer( - input_dict['src'], return_tensors='pt', - padding=True)['input_ids'] - ] - if self.eval_mode == EvaluationMode.REF or self.eval_mode == EvaluationMode.SRC_REF: - output_sents += [ - self.tokenizer( - input_dict['ref'], return_tensors='pt', - padding=True)['input_ids'] - ] + if (self.input_format == InputFormat.SRC + or self.input_format == InputFormat.SRC_REF) and (len( + input_dict['hyp']) != len(input_dict['src'])): + raise ValueError( + 'The number of given hyp sentences (%d) is not equal to that of src (%d).' + % (len(input_dict['hyp']), len(input_dict['src']))) + if (self.input_format == InputFormat.REF + or self.input_format == InputFormat.SRC_REF) and (len( + input_dict['hyp']) != len(input_dict['ref'])): + raise ValueError( + 'The number of given hyp sentences (%d) is not equal to that of ref (%d).' + % (len(input_dict['hyp']), len(input_dict['ref']))) - return output_sents + output_dict = {'input_ids': self.collect_input_ids(input_dict)} + + if self.mode == ModeKeys.TRAIN or self.mode == ModeKeys.EVAL: + if 'score' not in input_dict.keys(): + raise KeyError( + 'During training or evaluating, \'score\' should be provided.' + ) + if (isinstance(input_dict['score'], List) and len(input_dict['score']) != len(output_dict['input_ids'])) \ + or (isinstance(input_dict['score'], float) and len(output['input_ids']) != 1): + raise ValueError( + 'The number of score is not equal to that of the given examples. ' + 'Required %d, given %d.' % + (len(output['input_ids']), len(input_dict['score']))) + + output_dict['score'] = [input_dict['score']] if isinstance( + input_dict['score'], float) else input_dict['score'] + + if self.mode == ModeKeys.EVAL: + if 'lp' not in input_dict.keys(): + raise ValueError( + 'Language pair should be provided for evaluation.') + + if 'segment_id' not in input_dict.keys(): + raise ValueError( + 'Segment id should be provided for evaluation.') + + if 'raw_score' not in input_dict.keys(): + raise ValueError( + 'Raw scores should be provided for evaluation.') + + output_dict['lp'] = input_dict['lp'] + output_dict['segment_id'] = input_dict['segment_id'] + output_dict['raw_score'] = input_dict['raw_score'] + + return output_dict diff --git a/modelscope/trainers/nlp/__init__.py b/modelscope/trainers/nlp/__init__.py index 755e5387..ae102efa 100644 --- a/modelscope/trainers/nlp/__init__.py +++ b/modelscope/trainers/nlp/__init__.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from .text_generation_trainer import TextGenerationTrainer from .sentence_embedding_trainer import SentenceEmbeddingTrainer from .siamese_uie_trainer import SiameseUIETrainer + from .translation_evaluation_trainer import TranslationEvaluationTrainer else: _import_structure = { 'sequence_classification_trainer': ['SequenceClassificationTrainer'], @@ -17,7 +18,8 @@ else: 'text_ranking_trainer': ['TextRankingTrainer'], 'text_generation_trainer': ['TextGenerationTrainer'], 'sentence_emebedding_trainer': ['SentenceEmbeddingTrainer'], - 'siamese_uie_trainer': ['SiameseUIETrainer'] + 'siamese_uie_trainer': ['SiameseUIETrainer'], + 'translation_evaluation_trainer': ['TranslationEvaluationTrainer'] } import sys diff --git a/modelscope/trainers/nlp/translation_evaluation_trainer.py b/modelscope/trainers/nlp/translation_evaluation_trainer.py new file mode 100644 index 00000000..05e9db89 --- /dev/null +++ b/modelscope/trainers/nlp/translation_evaluation_trainer.py @@ -0,0 +1,396 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +"""PyTorch trainer for UniTE model.""" + +import os.path as osp +import random +from math import ceil +from os import mkdir +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +from pandas import DataFrame +from torch.nn.functional import pad +from torch.nn.utils import clip_grad_norm_ +from torch.optim import AdamW, Optimizer +from torch.utils.data import (BatchSampler, DataLoader, Dataset, Sampler, + SequentialSampler, SubsetRandomSampler) +from torch.utils.tensorboard import SummaryWriter +from tqdm import tqdm +from transformers import AutoTokenizer + +from modelscope.metainfo import Metrics, Trainers +from modelscope.metrics import Metric +from modelscope.metrics.builder import MetricKeys, build_metric +from modelscope.models.base import TorchModel +from modelscope.models.nlp.unite.configuration import InputFormat +from modelscope.models.nlp.unite.translation_evaluation import ( + UniTEForTranslationEvaluation, combine_input_sentences) +from modelscope.msdatasets import MsDataset +from modelscope.preprocessors import Preprocessor +from modelscope.trainers.builder import TRAINERS +from modelscope.trainers.hooks import Hook +from modelscope.trainers.trainer import EpochBasedTrainer +from modelscope.utils.config import ConfigDict +from modelscope.utils.constant import (ConfigKeys, Fields, ModeKeys, ModelFile, + TrainerStages) +from modelscope.utils.device import create_device +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +class TranslationEvaluationTrainingSampler(Sampler): + + def __init__(self, num_of_samples: int, + batch_size_for_each_input_format: int): + r"""Build a sampler for model training with translation evaluation trainer. + The trainer should derive samples for each subset of the entire dataset. + + Args: + num_of_samples: The number of samples in total. + batch_size_for_each_input_format: During training, the batch size for each input format + + Returns: + A data sampler for translation evaluation model training. + + """ + + self.num_of_samples = num_of_samples + self.batch_size_for_each_input_format = batch_size_for_each_input_format + + self.num_of_samples_for_each_input_format = self.num_of_samples // 3 + num_of_samples_to_use = self.num_of_samples_for_each_input_format * 3 + + logger.info( + '%d samples are given for training. ' + 'Using %d samples for each input format. ' + 'Leaving the last %d samples unused.' % + (self.num_of_samples, self.num_of_samples_for_each_input_format, + self.num_of_samples - num_of_samples_to_use)) + self.num_of_samples = num_of_samples_to_use + + random_permutations = torch.randperm( + self.num_of_samples).cpu().tolist() + + self.subset_iterators = dict() + self.subset_samplers = dict() + self.indices_for_each_input_format = dict() + for input_format_index, input_format in \ + enumerate((InputFormat.SRC_REF, InputFormat.SRC, InputFormat.REF)): + start_idx = input_format_index * self.num_of_samples_for_each_input_format + end_idx = start_idx + self.num_of_samples_for_each_input_format + self.indices_for_each_input_format[ + input_format] = random_permutations[start_idx:end_idx] + self.subset_samplers[input_format] = \ + BatchSampler(SubsetRandomSampler(self.indices_for_each_input_format[input_format]), + batch_size=self.batch_size_for_each_input_format, + drop_last=True) + self.subset_iterators[input_format] = iter( + self.subset_samplers[input_format]) + + self.num_of_sampled_batches = 0 + + if self.__len__() == 0: + raise ValueError( + 'The dataset doesn\'t contain enough examples to form a single batch.', + 'Please reduce the batch_size or use more examples for training.' + ) + + return + + def __iter__(self): + while True: + try: + if self.num_of_sampled_batches == self.__len__(): + for input_format in (InputFormat.SRC_REF, InputFormat.SRC, + InputFormat.REF): + while True: + try: + next(self.subset_iterators[input_format]) + except StopIteration: + self.subset_iterators[input_format] = \ + iter(self.subset_samplers[input_format]) + break + + self.num_of_sampled_batches = 0 + + output = list() + for input_format_idx, input_format in \ + enumerate((InputFormat.SRC_REF, InputFormat.SRC, InputFormat.REF)): + output += next(self.subset_iterators[input_format]) + + self.num_of_sampled_batches += 1 + + yield output + except StopIteration: + break + + def __len__(self) -> int: + return self.num_of_samples_for_each_input_format // self.batch_size_for_each_input_format + + +def convert_csv_dict_to_input( + batch: List[Dict[str, Any]], + preprocessor: Preprocessor) -> Tuple[List[torch.Tensor]]: + + input_dict = dict() + + for key in batch[0].keys(): + input_dict[key] = list(x[key] for x in batch) + + input_dict = preprocessor(input_dict) + + return input_dict + + +def data_collate_fn(batch: List[Dict[str, Any]], batch_size: int, + preprocessor: Preprocessor) -> List[Dict[str, Any]]: + + output_dict = dict() + output_dict['input_format'] = list() + + if preprocessor.mode == ModeKeys.TRAIN: + for input_format_index, input_format in \ + enumerate((InputFormat.SRC_REF, InputFormat.SRC, InputFormat.REF)): + start_idx = input_format_index * batch_size + end_idx = start_idx + batch_size + batch_to_process = batch[start_idx:end_idx] + output_dict['input_format'] += [input_format] * batch_size + preprocessor.change_input_format(input_format) + batch_to_process = convert_csv_dict_to_input( + batch_to_process, preprocessor) + + for key, value in batch_to_process.items(): + if key not in output_dict.keys(): + output_dict[key] = list() + output_dict[key].append(value) + elif preprocessor.mode == ModeKeys.EVAL: + output_dict['input_format'] += [preprocessor.input_format] * len(batch) + batch = convert_csv_dict_to_input(batch, preprocessor) + + for key, value in batch.items(): + if key not in output_dict.keys(): + output_dict[key] = list() + output_dict[key].append(value) + else: + raise ValueError( + 'During training, %s mode is not allowed for preprocessor.' + % preprocessor.mode) + + input_max_lengths = max(x.size(-1) for x in output_dict['input_ids']) + output_dict['input_ids'] = list( + pad(x, + pad=(0, input_max_lengths - x.size(-1)), + value=preprocessor.pad_token_id) for x in output_dict['input_ids']) + + output_dict['input_ids'] = torch.cat(output_dict['input_ids'], dim=0) + output_dict['score'] = torch.Tensor(output_dict['score']).view(-1) + + if preprocessor.mode == ModeKeys.EVAL: + output_dict['lp'] = sum(output_dict['lp'], list()) + output_dict['raw_score'] = sum(output_dict['raw_score'], list()) + output_dict['segment_id'] = sum(output_dict['segment_id'], list()) + + return output_dict + + +@TRAINERS.register_module(module_name=Trainers.translation_evaluation_trainer) +class TranslationEvaluationTrainer(EpochBasedTrainer): + + def __init__(self, + model: Optional[Union[TorchModel, torch.nn.Module, + str]] = None, + cfg_file: Optional[str] = None, + device: str = 'gpu', + *args, + **kwargs): + r"""Build a translation evaluation trainer with a model dir or a model id in the model hub. + + Args: + model: A Model instance. + cfg_file: The path for the configuration file (configuration.json). + device: Used device for this trainer. + + """ + + def data_collator_for_train(x): + return data_collate_fn( + x, + batch_size=self.cfg.train.batch_size, + preprocessor=self.train_preprocessor) + + def data_collator_for_eval(x): + return data_collate_fn( + x, + batch_size=self.cfg.evaluation.batch_size, + preprocessor=self.eval_preprocessor) + + data_collator = { + ConfigKeys.train: data_collator_for_train, + ConfigKeys.val: data_collator_for_eval + } + + super().__init__( + model, + cfg_file=cfg_file, + data_collator=data_collator, + *args, + **kwargs) + + self.train_dataloader = None + self.eval_dataloader = None + + return + + def build_optimizer(self, cfg: ConfigDict) -> Optimizer: + r"""Sets the optimizers to be used during training.""" + if self.cfg.train.optimizer.type != 'AdamW': + return super().build_optimizer(cfg) + + # Freezing embedding layers for more efficient training. + for param in self.model.encoder.embeddings.parameters(): + param.requires_grad = False + + logger.info('Building AdamW optimizer ...') + learning_rates_and_parameters = list({ + 'params': + self.model.encoder.encoder.layer[i].parameters(), + 'lr': + self.cfg.train.optimizer.plm_lr + * self.cfg.train.optimizer.plm_lr_layerwise_decay**i, + } for i in range(0, self.cfg.model.num_hidden_layers)) + + learning_rates_and_parameters.append({ + 'params': + self.model.encoder.embeddings.parameters(), + 'lr': + self.cfg.train.optimizer.plm_lr, + }) + + learning_rates_and_parameters.append({ + 'params': + self.model.estimator.parameters(), + 'lr': + self.cfg.train.optimizer.mlp_lr + }) + + learning_rates_and_parameters.append({ + 'params': + self.model.layerwise_attention.parameters(), + 'lr': + self.cfg.train.optimizer.mlp_lr, + }) + + optimizer = AdamW( + learning_rates_and_parameters, + lr=self.cfg.train.optimizer.plm_lr, + betas=self.cfg.train.optimizer.betas, + eps=self.cfg.train.optimizer.eps, + weight_decay=self.cfg.train.optimizer.weight_decay, + ) + + return optimizer + + def get_train_dataloader(self) -> DataLoader: + logger.info('Building dataloader for training ...') + + if self.train_dataset is None: + logger.info('Reading train csv file from %s ...' + % self.cfg.dataset.train.name) + self.train_dataset = MsDataset.load( + osp.join(self.model_dir, self.cfg.dataset.train.name), + split=self.cfg.dataset.train.split) + + train_dataloader = DataLoader( + self.train_dataset, + batch_sampler=TranslationEvaluationTrainingSampler( + len(self.train_dataset), + batch_size_for_each_input_format=self.cfg.train.batch_size), + num_workers=4, + collate_fn=self.train_data_collator, + generator=None) + + logger.info('Reading done, %d items in total' + % len(self.train_dataset)) + + return train_dataloader + + def get_eval_data_loader(self) -> DataLoader: + logger.info('Building dataloader for evaluating ...') + + if self.eval_dataset is None: + logger.info('Reading eval csv file from %s ...' + % self.cfg.dataset.valid.name) + + self.eval_dataset = MsDataset.load( + osp.join(self.model_dir, self.cfg.dataset.valid.name), + split=self.cfg.dataset.valid.split) + + eval_dataloader = DataLoader( + self.eval_dataset, + batch_sampler=BatchSampler( + SequentialSampler(range(0, len(self.eval_dataset))), + batch_size=self.cfg.evaluation.batch_size, + drop_last=False), + num_workers=4, + collate_fn=self.eval_data_collator, + generator=None) + + logger.info('Reading done, %d items in total' % len(self.eval_dataset)) + + return eval_dataloader + + def evaluation_loop(self, data_loader, metric_classes): + """ Evaluation loop used by `TranslationEvaluationTrainer.evaluate()`. + + The evaluation process of UniTE model should be arranged with three loops, + corresponding to the input formats of `InputFormat.SRC_REF`, `InputFormat.REF`, + and `InputFormat.SRC`. + + Here we directly copy the codes of `EpochBasedTrainer.evaluation_loop`, and change + the input format during each evaluation subloop. + """ + vis_closure = None + if hasattr(self.cfg.evaluation, 'visualization'): + vis_cfg = self.cfg.evaluation.visualization + vis_closure = partial( + self.visualization, dataset=self.eval_dataset, **vis_cfg) + + self.invoke_hook(TrainerStages.before_val) + metric_values = dict() + + for input_format in (InputFormat.SRC_REF, InputFormat.SRC, + InputFormat.REF): + self.eval_preprocessor.change_input_format(input_format) + + if self._dist: + from modelscope.trainers.utils.inference import multi_gpu_test + # list of batched result and data samples + metric_values.update( + multi_gpu_test( + self, + data_loader, + device=self.device, + metric_classes=metric_classes, + vis_closure=vis_closure, + tmpdir=self.cfg.evaluation.get('cache_dir', None), + gpu_collect=self.cfg.evaluation.get( + 'gpu_collect', False), + data_loader_iters_per_gpu=self._eval_iters_per_epoch)) + else: + from modelscope.trainers.utils.inference import single_gpu_test + metric_values.update( + single_gpu_test( + self, + data_loader, + device=self.device, + metric_classes=metric_classes, + vis_closure=vis_closure, + data_loader_iters=self._eval_iters_per_epoch)) + + for m in metric_classes: + if hasattr(m, 'clear') and callable(m.clear): + m.clear() + + self.invoke_hook(TrainerStages.after_val) + return metric_values diff --git a/tests/metrics/test_translation_evaluation_metrics.py b/tests/metrics/test_translation_evaluation_metrics.py new file mode 100644 index 00000000..801f742b --- /dev/null +++ b/tests/metrics/test_translation_evaluation_metrics.py @@ -0,0 +1,30 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import unittest + +from modelscope.metrics.translation_evaluation_metric import \ + TranslationEvaluationMetric +from modelscope.models.nlp.unite.configuration import InputFormat +from modelscope.utils.test_utils import test_level + + +class TestTranslationEvaluationMetrics(unittest.TestCase): + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_value(self): + metric = TranslationEvaluationMetric(gap_threshold=25.0) + + outputs = {'score': [0.25, 0.22, 0.30, 0.78, 1.11, 0.95, 1.00, 0.86]} + inputs = { + 'lp': ['zh-en'] * 8, + 'segment_id': [0, 0, 0, 1, 1, 2, 2, 2], + 'raw_score': [94.0, 60.0, 25.0, 59.5, 90.0, 100.0, 80.0, 60.0], + 'input_format': [InputFormat.SRC_REF] * 8, + } + metric.add(outputs, inputs) + result = metric.evaluate() + print(result) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/pipelines/test_translation_evaluation.py b/tests/pipelines/test_translation_evaluation.py index 53524fdc..8f16ea31 100644 --- a/tests/pipelines/test_translation_evaluation.py +++ b/tests/pipelines/test_translation_evaluation.py @@ -2,7 +2,7 @@ import unittest -from modelscope.models.nlp.unite.configuration_unite import EvaluationMode +from modelscope.models.nlp.unite.configuration import InputFormat from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.demo_utils import DemoCompatibilityCheck @@ -18,7 +18,7 @@ class TranslationEvaluationTest(unittest.TestCase, DemoCompatibilityCheck): @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_run_with_model_name_for_unite_large(self): - input = { + input_dict = { 'hyp': [ 'This is a sentence.', 'This is another sentence.', @@ -34,27 +34,27 @@ class TranslationEvaluationTest(unittest.TestCase, DemoCompatibilityCheck): } pipeline_ins = pipeline(self.task, model=self.model_id_large) - print(pipeline_ins(input=input)) + print(pipeline_ins(input_dict)['score']) - pipeline_ins.change_eval_mode(eval_mode=EvaluationMode.SRC) - print(pipeline_ins(input=input)) + pipeline_ins.change_input_format(input_format=InputFormat.SRC) + print(pipeline_ins(input_dict)['score']) - pipeline_ins.change_eval_mode(eval_mode=EvaluationMode.REF) - print(pipeline_ins(input=input)) + pipeline_ins.change_input_format(input_format=InputFormat.REF) + print(pipeline_ins(input_dict)['score']) pipeline_ins = pipeline( self.task, model=self.model_id_large, device='cpu') - print(pipeline_ins(input=input)) + print(pipeline_ins(input_dict)['score']) - pipeline_ins.change_eval_mode(eval_mode=EvaluationMode.SRC) - print(pipeline_ins(input=input)) + pipeline_ins.change_input_format(input_format=InputFormat.SRC) + print(pipeline_ins(input_dict)['score']) - pipeline_ins.change_eval_mode(eval_mode=EvaluationMode.REF) - print(pipeline_ins(input=input)) + pipeline_ins.change_input_format(input_format=InputFormat.REF) + print(pipeline_ins(input_dict)['score']) @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_run_with_model_name_for_unite_base(self): - input = { + input_dict = { 'hyp': [ 'This is a sentence.', 'This is another sentence.', @@ -70,23 +70,23 @@ class TranslationEvaluationTest(unittest.TestCase, DemoCompatibilityCheck): } pipeline_ins = pipeline(self.task, model=self.model_id_base) - print(pipeline_ins(input=input)) + print(pipeline_ins(input_dict)['score']) - pipeline_ins.change_eval_mode(eval_mode=EvaluationMode.SRC) - print(pipeline_ins(input=input)) + pipeline_ins.change_input_format(input_format=InputFormat.SRC) + print(pipeline_ins(input_dict)['score']) - pipeline_ins.change_eval_mode(eval_mode=EvaluationMode.REF) - print(pipeline_ins(input=input)) + pipeline_ins.change_input_format(input_format=InputFormat.REF) + print(pipeline_ins(input_dict)['score']) pipeline_ins = pipeline( self.task, model=self.model_id_base, device='cpu') - print(pipeline_ins(input=input)) + print(pipeline_ins(input_dict)['score']) - pipeline_ins.change_eval_mode(eval_mode=EvaluationMode.SRC) - print(pipeline_ins(input=input)) + pipeline_ins.change_input_format(input_format=InputFormat.SRC) + print(pipeline_ins(input_dict)['score']) - pipeline_ins.change_eval_mode(eval_mode=EvaluationMode.REF) - print(pipeline_ins(input=input)) + pipeline_ins.change_input_format(input_format=InputFormat.REF) + print(pipeline_ins(input_dict)['score']) if __name__ == '__main__': diff --git a/tests/run_config.yaml b/tests/run_config.yaml index 773c6397..14de4c85 100644 --- a/tests/run_config.yaml +++ b/tests/run_config.yaml @@ -21,6 +21,7 @@ isolated: # test cases that may require excessive anmount of GPU memory or run - test_image_instance_segmentation_trainer.py - test_image_portrait_enhancement_trainer.py - test_translation_trainer.py + - test_translation_evaluation_trainer.py - test_unifold.py - test_automatic_post_editing.py - test_mplug_tasks.py @@ -77,6 +78,7 @@ envs: - test_text_to_speech.py - test_csanmt_translation.py - test_translation_trainer.py + - test_translation_evaluation_trainer.py - test_ocr_detection.py - test_automatic_speech_recognition.py - test_image_matting.py diff --git a/tests/trainers/model_trainer_map.py b/tests/trainers/model_trainer_map.py index d4c4c09a..4e9005f7 100644 --- a/tests/trainers/model_trainer_map.py +++ b/tests/trainers/model_trainer_map.py @@ -124,6 +124,12 @@ model_trainer_map = { 'damo/nlp_csanmt_translation_en2es': [ 'tests/trainers/test_translation_trainer.py' ], + 'damo/nlp_unite_mup_translation_evaluation_multilingual_base': [ + 'tests/trainers/test_translation_evaluation_trainer.py' + ], + 'damo/nlp_unite_mup_translation_evaluation_multilingual_large': [ + 'tests/trainers/test_translation_evaluation_trainer.py' + ], 'damo/cv_googlenet_pgl-video-summarization': [ 'tests/trainers/test_video_summarization_trainer.py' ], diff --git a/tests/trainers/test_translation_evaluation_trainer.py b/tests/trainers/test_translation_evaluation_trainer.py new file mode 100644 index 00000000..139427da --- /dev/null +++ b/tests/trainers/test_translation_evaluation_trainer.py @@ -0,0 +1,30 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import unittest + +from modelscope.metainfo import Trainers +from modelscope.trainers import build_trainer +from modelscope.utils.test_utils import test_level + + +class TranslationEvaluationTest(unittest.TestCase): + + def setUp(self) -> None: + self.name = Trainers.translation_evaluation_trainer + self.model_id_large = 'damo/nlp_unite_mup_translation_evaluation_multilingual_large' + self.model_id_base = 'damo/nlp_unite_mup_translation_evaluation_multilingual_base' + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_unite_mup_large(self) -> None: + default_args = {'model': self.model_id_large} + trainer = build_trainer(name=self.name, default_args=default_args) + trainer.train() + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_unite_mup_base(self) -> None: + default_args = {'model': self.model_id_base} + trainer = build_trainer(name=self.name, default_args=default_args) + trainer.train() + + +if __name__ == '__main__': + unittest.main() From 49339941dfedc0f1cc0b00c242d4d7e12f928181 Mon Sep 17 00:00:00 2001 From: "hemu.zp" Date: Thu, 11 May 2023 15:50:30 +0800 Subject: [PATCH 22/76] Add convert megatron ckpt script Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12555222 * add convert megatron ckpt script --- modelscope/utils/megatron_utils.py | 3 ++- tools/convert_megatron_ckpt.py | 31 ++++++++++++++++++++++++++++++ tools/convert_megatron_ckpt.sh | 7 +++++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 tools/convert_megatron_ckpt.py create mode 100644 tools/convert_megatron_ckpt.sh diff --git a/modelscope/utils/megatron_utils.py b/modelscope/utils/megatron_utils.py index 922cb53d..53b5aacb 100644 --- a/modelscope/utils/megatron_utils.py +++ b/modelscope/utils/megatron_utils.py @@ -96,15 +96,16 @@ def convert_megatron_checkpoint( log_master( f'origin_num_partitions: {origin_num_partitions}, target_num_partitions: {target_num_partitions}' ) - os.makedirs(target_dir, exist_ok=True) if origin_num_partitions < target_num_partitions: + os.makedirs(target_dir, exist_ok=True) state_dict = _split_checkpoint( model, checkpoint_dir, target_num_partitions // origin_num_partitions) _save_converted_checkpoint(state_dict, target_dir) log_master('Split checkpoints succeeded.') elif origin_num_partitions > target_num_partitions: + os.makedirs(target_dir, exist_ok=True) state_dict = _merge_checkpoint( model, checkpoint_dir, origin_num_partitions // target_num_partitions) diff --git a/tools/convert_megatron_ckpt.py b/tools/convert_megatron_ckpt.py new file mode 100644 index 00000000..f9b8f8f3 --- /dev/null +++ b/tools/convert_megatron_ckpt.py @@ -0,0 +1,31 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import argparse +import os + +from modelscope.models import Model +from modelscope.utils.megatron_utils import convert_megatron_checkpoint + + +def unwrap_model(model): + for name in ('model', 'module', 'dist_model'): + while hasattr(model, name): + model = getattr(model, name) + return model + + +parser = argparse.ArgumentParser( + description='Split or merge your megatron_based checkpoint.') +parser.add_argument( + '--model_dir', type=str, required=True, help='Checkpoint to be converted.') +parser.add_argument( + '--target_dir', type=str, required=True, help='Target save path.') +args = parser.parse_args() + +model = Model.from_pretrained( + args.model_dir, + rank=int(os.getenv('RANK')), + megatron_cfg={'tensor_model_parallel_size': int(os.getenv('WORLD_SIZE'))}) +unwrapped_model = unwrap_model(model) + +convert_megatron_checkpoint(unwrapped_model, model.model_dir, args.target_dir) diff --git a/tools/convert_megatron_ckpt.sh b/tools/convert_megatron_ckpt.sh new file mode 100644 index 00000000..86e94877 --- /dev/null +++ b/tools/convert_megatron_ckpt.sh @@ -0,0 +1,7 @@ +TARGET_TENSOR_MODEL_PARALLEL_SIZE=1 +ORIGIN_MODEL='damo/nlp_gpt3_text-generation_1.3B' +TARGET_DIR='./target' + +torchrun --nproc_per_node $TARGET_TENSOR_MODEL_PARALLEL_SIZE tools/convert_megatron_ckpt.py \ + --model_dir $ORIGIN_MODEL \ + --target_dir $TARGET_DIR \ From 4c0b13d15737149541e564bedc72e8356226d554 Mon Sep 17 00:00:00 2001 From: "wucong.lyb" Date: Thu, 11 May 2023 20:13:20 +0800 Subject: [PATCH 23/76] fix func update_local_model; change funasr version Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12595406 --- modelscope/utils/audio/audio_utils.py | 50 +++++++++++++-------------- requirements/audio/audio_asr.txt | 2 +- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/modelscope/utils/audio/audio_utils.py b/modelscope/utils/audio/audio_utils.py index 3336af06..562769b8 100644 --- a/modelscope/utils/audio/audio_utils.py +++ b/modelscope/utils/audio/audio_utils.py @@ -11,6 +11,7 @@ from urllib.parse import urlparse import numpy as np from modelscope.fileio.file import HTTPStorage +from modelscope.hub.utils.utils import get_cache_dir from modelscope.utils.hub import snapshot_download from modelscope.utils.logger import get_logger @@ -323,34 +324,31 @@ def generate_sd_scp_from_url(urls: Union[tuple, list]): def update_local_model(model_config, model_path, extra_args): + if 'update_model' in extra_args and not extra_args['update_model']: + return + model_revision = None if 'update_model' in extra_args: if extra_args['update_model'] == 'latest': model_revision = None else: model_revision = extra_args['update_model'] - if model_config.__contains__('model'): - model_name = model_config['model'] - if isinstance(model_path, str) and os.path.exists(model_path): - try: - logger.info( - 'Download the model to local path {0} ...'.format( - model_path)) - src_path = snapshot_download( - model_name, revision=model_revision) - # cp to model_path - if src_path == model_path: - logger.warning('src_path is the same with model_path') - return - for filename in os.listdir(src_path): - src_file = os.path.join(src_path, filename) - dst_file = os.path.join(model_path, filename) - if os.path.isfile(src_file): - shutil.copy2(src_file, model_path) - elif os.path.isdir(src_file): - if os.path.exists(dst_file): - shutil.rmtree(dst_file) - shutil.copytree(src_file, dst_file) - except Exception as e: - logger.warning(str(e)) - else: - logger.warning('Can not find model name in configuration') + if model_config.__contains__('model'): + model_name = model_config['model'] + dst_dir_root = get_cache_dir() + if isinstance(model_path, str) and os.path.exists( + model_path) and not model_path.startswith(dst_dir_root): + try: + dst = os.path.join(dst_dir_root, '.cache/' + model_name) + dst_dir = os.path.dirname(dst) + os.makedirs(dst_dir, exist_ok=True) + if not os.path.exists(dst): + os.symlink(os.path.abspath(model_path), dst) + + snapshot_download( + model_name, + cache_dir=dst_dir_root, + revision=model_revision) + except Exception as e: + logger.warning(str(e)) + else: + logger.warning('Can not find model name in configuration') diff --git a/requirements/audio/audio_asr.txt b/requirements/audio/audio_asr.txt index 1ecbe421..7725a0dd 100644 --- a/requirements/audio/audio_asr.txt +++ b/requirements/audio/audio_asr.txt @@ -1,2 +1,2 @@ easyasr>=0.0.2 -funasr>=0.4.0 +funasr>=0.5.0 From c16a3b847fe7ecbd6d59ee444b64c2b888dec267 Mon Sep 17 00:00:00 2001 From: "zhifu.gzf" Date: Fri, 12 May 2023 09:02:05 +0800 Subject: [PATCH 24/76] audio funasr pipeline kwargs --- modelscope/pipelines/audio/asr_inference_pipeline.py | 4 +++- modelscope/pipelines/audio/lm_infer_pipeline.py | 11 ++++++++--- .../audio/punctuation_processing_pipeline.py | 11 ++++++++--- .../pipelines/audio/speaker_diarization_pipeline.py | 4 +++- .../pipelines/audio/speaker_verification_pipeline.py | 8 ++++++-- modelscope/pipelines/audio/timestamp_pipeline.py | 11 ++++++++--- .../audio/voice_activity_detection_pipeline.py | 11 ++++++++--- 7 files changed, 44 insertions(+), 16 deletions(-) diff --git a/modelscope/pipelines/audio/asr_inference_pipeline.py b/modelscope/pipelines/audio/asr_inference_pipeline.py index 6387bb3a..aa37c9e3 100644 --- a/modelscope/pipelines/audio/asr_inference_pipeline.py +++ b/modelscope/pipelines/audio/asr_inference_pipeline.py @@ -54,6 +54,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): lm_model_revision: Optional[str] = None, timestamp_model: Optional[Union[Model, str]] = None, timestamp_model_revision: Optional[str] = None, + ngpu: int = 1, **kwargs): """ Use `model` and `preprocessor` to create an asr pipeline for prediction @@ -127,7 +128,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): minlenratio=self.cmd['minlenratio'], batch_size=self.cmd['batch_size'], beam_size=self.cmd['beam_size'], - ngpu=self.cmd['ngpu'], + ngpu=ngpu, ctc_weight=self.cmd['ctc_weight'], lm_weight=self.cmd['lm_weight'], penalty=self.cmd['penalty'], @@ -160,6 +161,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): token_num_relax=self.cmd['token_num_relax'], decoding_ind=self.cmd['decoding_ind'], decoding_mode=self.cmd['decoding_mode'], + **kwargs, ) def __call__(self, diff --git a/modelscope/pipelines/audio/lm_infer_pipeline.py b/modelscope/pipelines/audio/lm_infer_pipeline.py index f271ea45..5e8928e5 100644 --- a/modelscope/pipelines/audio/lm_infer_pipeline.py +++ b/modelscope/pipelines/audio/lm_infer_pipeline.py @@ -35,7 +35,10 @@ class LanguageModelPipeline(Pipeline): """ - def __init__(self, model: Union[Model, str] = None, **kwargs): + def __init__(self, + model: Union[Model, str] = None, + ngpu: int = 1, + **kwargs): """ Use `model` to create a LM pipeline for prediction Args: @@ -77,7 +80,7 @@ class LanguageModelPipeline(Pipeline): mode=self.cmd['mode'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=self.cmd['ngpu'], + ngpu=ngpu, seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], @@ -88,7 +91,9 @@ class LanguageModelPipeline(Pipeline): split_with_space=self.cmd['split_with_space'], seg_dict_file=self.cmd['seg_dict_file'], output_dir=self.cmd['output_dir'], - param_dict=self.cmd['param_dict']) + param_dict=self.cmd['param_dict'], + **kwargs, + ) def __call__(self, text_in: str = None, diff --git a/modelscope/pipelines/audio/punctuation_processing_pipeline.py b/modelscope/pipelines/audio/punctuation_processing_pipeline.py index 2f4dee7a..54a26a8a 100644 --- a/modelscope/pipelines/audio/punctuation_processing_pipeline.py +++ b/modelscope/pipelines/audio/punctuation_processing_pipeline.py @@ -39,7 +39,10 @@ class PunctuationProcessingPipeline(Pipeline): """ - def __init__(self, model: Union[Model, str] = None, **kwargs): + def __init__(self, + model: Union[Model, str] = None, + ngpu: int = 1, + **kwargs): """use `model` to create an asr pipeline for prediction """ super().__init__(model=model, **kwargs) @@ -51,7 +54,7 @@ class PunctuationProcessingPipeline(Pipeline): mode=self.cmd['mode'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=self.cmd['ngpu'], + ngpu=ngpu, seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], @@ -59,7 +62,9 @@ class PunctuationProcessingPipeline(Pipeline): train_config=self.cmd['train_config'], model_file=self.cmd['model_file'], output_dir=self.cmd['output_dir'], - param_dict=self.cmd['param_dict']) + param_dict=self.cmd['param_dict'], + **kwargs, + ) def __call__(self, text_in: str = None, diff --git a/modelscope/pipelines/audio/speaker_diarization_pipeline.py b/modelscope/pipelines/audio/speaker_diarization_pipeline.py index f4f68cba..af6c81e8 100644 --- a/modelscope/pipelines/audio/speaker_diarization_pipeline.py +++ b/modelscope/pipelines/audio/speaker_diarization_pipeline.py @@ -48,6 +48,7 @@ class SpeakerDiarizationPipeline(Pipeline): model: Union[Model, str] = None, sv_model: Optional[Union[Model, str]] = None, sv_model_revision: Optional[str] = None, + ngpu: int = 1, **kwargs): """use `model` to create a speaker diarization pipeline for prediction Args: @@ -76,7 +77,7 @@ class SpeakerDiarizationPipeline(Pipeline): output_dir=self.cmd['output_dir'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=self.cmd['ngpu'], + ngpu=ngpu, seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], @@ -90,6 +91,7 @@ class SpeakerDiarizationPipeline(Pipeline): dur_threshold=self.cmd['dur_threshold'], out_format=self.cmd['out_format'], param_dict=self.cmd['param_dict'], + **kwargs, ) def __call__(self, diff --git a/modelscope/pipelines/audio/speaker_verification_pipeline.py b/modelscope/pipelines/audio/speaker_verification_pipeline.py index 97e73627..2f68cf95 100644 --- a/modelscope/pipelines/audio/speaker_verification_pipeline.py +++ b/modelscope/pipelines/audio/speaker_verification_pipeline.py @@ -41,7 +41,10 @@ class SpeakerVerificationPipeline(Pipeline): """ - def __init__(self, model: Union[Model, str] = None, **kwargs): + def __init__(self, + model: Union[Model, str] = None, + ngpu: int = 1, + **kwargs): """use `model` to create an asr pipeline for prediction """ super().__init__(model=model, **kwargs) @@ -54,7 +57,7 @@ class SpeakerVerificationPipeline(Pipeline): output_dir=self.cmd['output_dir'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=self.cmd['ngpu'], + ngpu=ngpu, seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], @@ -67,6 +70,7 @@ class SpeakerVerificationPipeline(Pipeline): embedding_node=self.cmd['embedding_node'], sv_threshold=self.cmd['sv_threshold'], param_dict=self.cmd['param_dict'], + **kwargs, ) def __call__(self, diff --git a/modelscope/pipelines/audio/timestamp_pipeline.py b/modelscope/pipelines/audio/timestamp_pipeline.py index b60fef05..13fb0f82 100644 --- a/modelscope/pipelines/audio/timestamp_pipeline.py +++ b/modelscope/pipelines/audio/timestamp_pipeline.py @@ -40,7 +40,10 @@ class TimestampPipeline(Pipeline): """ - def __init__(self, model: Union[Model, str] = None, **kwargs): + def __init__(self, + model: Union[Model, str] = None, + ngpu: int = 1, + **kwargs): """ Use `model` and `preprocessor` to create an asr pipeline for prediction Args: @@ -72,7 +75,7 @@ class TimestampPipeline(Pipeline): mode=self.cmd['mode'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=self.cmd['ngpu'], + ngpu=ngpu, seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], @@ -84,7 +87,9 @@ class TimestampPipeline(Pipeline): allow_variable_data_keys=self.cmd['allow_variable_data_keys'], split_with_space=self.cmd['split_with_space'], seg_dict_file=self.cmd['seg_dict_file'], - param_dict=self.cmd['param_dict']) + param_dict=self.cmd['param_dict'], + **kwargs, + ) def __call__(self, audio_in: Union[str, bytes], diff --git a/modelscope/pipelines/audio/voice_activity_detection_pipeline.py b/modelscope/pipelines/audio/voice_activity_detection_pipeline.py index c1c6e01f..f1bb86b6 100644 --- a/modelscope/pipelines/audio/voice_activity_detection_pipeline.py +++ b/modelscope/pipelines/audio/voice_activity_detection_pipeline.py @@ -41,7 +41,10 @@ class VoiceActivityDetectionPipeline(Pipeline): """ - def __init__(self, model: Union[Model, str] = None, **kwargs): + def __init__(self, + model: Union[Model, str] = None, + ngpu: int = 1, + **kwargs): """use `model` to create an vad pipeline for prediction """ super().__init__(model=model, **kwargs) @@ -53,14 +56,16 @@ class VoiceActivityDetectionPipeline(Pipeline): mode=self.cmd['mode'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=self.cmd['ngpu'], + ngpu=ngpu, seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], key_file=self.cmd['key_file'], vad_infer_config=self.cmd['vad_infer_config'], vad_model_file=self.cmd['vad_model_file'], - vad_cmvn_file=self.cmd['vad_cmvn_file']) + vad_cmvn_file=self.cmd['vad_cmvn_file'], + **kwargs, + ) def __call__(self, audio_in: Union[str, bytes], From f2e197ba915f10b3e0da6351a4074edb5ed08ad0 Mon Sep 17 00:00:00 2001 From: "zhifu.gzf" Date: Fri, 12 May 2023 10:39:22 +0800 Subject: [PATCH 25/76] rm easrasr --- data/test | 2 +- .../generic_automatic_speech_recognition.py | 22 --- .../pipelines/audio/asr_inference_pipeline.py | 179 +++++++----------- modelscope/preprocessors/asr.py | 67 ------- 4 files changed, 67 insertions(+), 203 deletions(-) diff --git a/data/test b/data/test index 8d062525..91b37f8d 160000 --- a/data/test +++ b/data/test @@ -1 +1 @@ -Subproject commit 8d0625256b88bdf41655563049a4a68ec1025638 +Subproject commit 91b37f8d6251089aa878520f389fbefc2ce9ffc4 diff --git a/modelscope/models/audio/asr/generic_automatic_speech_recognition.py b/modelscope/models/audio/asr/generic_automatic_speech_recognition.py index 25de839e..8dd11982 100644 --- a/modelscope/models/audio/asr/generic_automatic_speech_recognition.py +++ b/modelscope/models/audio/asr/generic_automatic_speech_recognition.py @@ -45,27 +45,5 @@ class GenericAutomaticSpeechRecognition(Model): def forward(self) -> Dict[str, Any]: """preload model and return the info of the model """ - if self.model_cfg['model_config']['type'] == Frameworks.tf: - from easyasr import asr_inference_paraformer_tf - if hasattr(asr_inference_paraformer_tf, 'preload'): - model_workspace = self.model_cfg['model_workspace'] - model_path = os.path.join(model_workspace, - self.model_cfg['am_model']) - vocab_path = os.path.join( - model_workspace, - self.model_cfg['model_config']['vocab_file']) - sampled_ids = 'seq2seq/sampled_ids' - sampled_lengths = 'seq2seq/sampled_lengths' - if 'sampled_ids' in self.model_cfg['model_config']: - sampled_ids = self.model_cfg['model_config']['sampled_ids'] - if 'sampled_lengths' in self.model_cfg['model_config']: - sampled_lengths = self.model_cfg['model_config'][ - 'sampled_lengths'] - asr_inference_paraformer_tf.preload( - ngpu=1, - asr_model_file=model_path, - vocab_file=vocab_path, - sampled_ids=sampled_ids, - sampled_lengths=sampled_lengths) return self.model_cfg diff --git a/modelscope/pipelines/audio/asr_inference_pipeline.py b/modelscope/pipelines/audio/asr_inference_pipeline.py index aa37c9e3..09764c69 100644 --- a/modelscope/pipelines/audio/asr_inference_pipeline.py +++ b/modelscope/pipelines/audio/asr_inference_pipeline.py @@ -120,49 +120,48 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): self.model_cfg = self.model.forward() self.cmd = self.get_cmd(kwargs, model) - if self.cmd['code_base'] == 'funasr': - from funasr.bin import asr_inference_launch - self.funasr_infer_modelscope = asr_inference_launch.inference_launch( - mode=self.cmd['mode'], - maxlenratio=self.cmd['maxlenratio'], - minlenratio=self.cmd['minlenratio'], - batch_size=self.cmd['batch_size'], - beam_size=self.cmd['beam_size'], - ngpu=ngpu, - ctc_weight=self.cmd['ctc_weight'], - lm_weight=self.cmd['lm_weight'], - penalty=self.cmd['penalty'], - log_level=self.cmd['log_level'], - asr_train_config=self.cmd['asr_train_config'], - asr_model_file=self.cmd['asr_model_file'], - cmvn_file=self.cmd['cmvn_file'], - lm_file=self.cmd['lm_file'], - token_type=self.cmd['token_type'], - key_file=self.cmd['key_file'], - lm_train_config=self.cmd['lm_train_config'], - bpemodel=self.cmd['bpemodel'], - allow_variable_data_keys=self.cmd['allow_variable_data_keys'], - output_dir=self.cmd['output_dir'], - dtype=self.cmd['dtype'], - seed=self.cmd['seed'], - ngram_weight=self.cmd['ngram_weight'], - nbest=self.cmd['nbest'], - num_workers=self.cmd['num_workers'], - vad_infer_config=self.cmd['vad_infer_config'], - vad_model_file=self.cmd['vad_model_file'], - vad_cmvn_file=self.cmd['vad_cmvn_file'], - punc_model_file=self.cmd['punc_model_file'], - punc_infer_config=self.cmd['punc_infer_config'], - timestamp_model_file=self.cmd['timestamp_model_file'], - timestamp_infer_config=self.cmd['timestamp_infer_config'], - timestamp_cmvn_file=self.cmd['timestamp_cmvn_file'], - outputs_dict=self.cmd['outputs_dict'], - param_dict=self.cmd['param_dict'], - token_num_relax=self.cmd['token_num_relax'], - decoding_ind=self.cmd['decoding_ind'], - decoding_mode=self.cmd['decoding_mode'], - **kwargs, - ) + from funasr.bin import asr_inference_launch + self.funasr_infer_modelscope = asr_inference_launch.inference_launch( + mode=self.cmd['mode'], + maxlenratio=self.cmd['maxlenratio'], + minlenratio=self.cmd['minlenratio'], + batch_size=self.cmd['batch_size'], + beam_size=self.cmd['beam_size'], + ngpu=ngpu, + ctc_weight=self.cmd['ctc_weight'], + lm_weight=self.cmd['lm_weight'], + penalty=self.cmd['penalty'], + log_level=self.cmd['log_level'], + asr_train_config=self.cmd['asr_train_config'], + asr_model_file=self.cmd['asr_model_file'], + cmvn_file=self.cmd['cmvn_file'], + lm_file=self.cmd['lm_file'], + token_type=self.cmd['token_type'], + key_file=self.cmd['key_file'], + lm_train_config=self.cmd['lm_train_config'], + bpemodel=self.cmd['bpemodel'], + allow_variable_data_keys=self.cmd['allow_variable_data_keys'], + output_dir=self.cmd['output_dir'], + dtype=self.cmd['dtype'], + seed=self.cmd['seed'], + ngram_weight=self.cmd['ngram_weight'], + nbest=self.cmd['nbest'], + num_workers=self.cmd['num_workers'], + vad_infer_config=self.cmd['vad_infer_config'], + vad_model_file=self.cmd['vad_model_file'], + vad_cmvn_file=self.cmd['vad_cmvn_file'], + punc_model_file=self.cmd['punc_model_file'], + punc_infer_config=self.cmd['punc_infer_config'], + timestamp_model_file=self.cmd['timestamp_model_file'], + timestamp_infer_config=self.cmd['timestamp_infer_config'], + timestamp_cmvn_file=self.cmd['timestamp_cmvn_file'], + outputs_dict=self.cmd['outputs_dict'], + param_dict=self.cmd['param_dict'], + token_num_relax=self.cmd['token_num_relax'], + decoding_ind=self.cmd['decoding_ind'], + decoding_mode=self.cmd['decoding_mode'], + **kwargs, + ) def __call__(self, audio_in: Union[str, bytes], @@ -199,7 +198,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): """ # code base - code_base = self.cmd['code_base'] + # code_base = self.cmd['code_base'] self.recog_type = recog_type self.audio_format = audio_format self.audio_fs = None @@ -209,31 +208,21 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): self.cmd['output_dir'] = output_dir self.cmd['param_dict'] = param_dict - if code_base == 'funasr': - if isinstance(audio_in, str): - # for funasr code, generate wav.scp from url or local path - self.audio_in, self.raw_inputs = generate_scp_from_url( - audio_in) - elif isinstance(audio_in, bytes): - self.audio_in = audio_in - self.raw_inputs = None - else: - import numpy - import torch - if isinstance(audio_in, torch.Tensor): - self.audio_in = None - self.raw_inputs = audio_in - elif isinstance(audio_in, numpy.ndarray): - self.audio_in = None - self.raw_inputs = audio_in - elif isinstance(audio_in, str): - # load pcm data from url if audio_in is url str - self.audio_in, checking_audio_fs = load_bytes_from_url(audio_in) + if isinstance(audio_in, str): + # for funasr code, generate wav.scp from url or local path + self.audio_in, self.raw_inputs = generate_scp_from_url(audio_in) elif isinstance(audio_in, bytes): - # load pcm data from wav data if audio_in is wave format - self.audio_in, checking_audio_fs = extract_pcm_from_wav(audio_in) - else: self.audio_in = audio_in + self.raw_inputs = None + else: + import numpy + import torch + if isinstance(audio_in, torch.Tensor): + self.audio_in = None + self.raw_inputs = audio_in + elif isinstance(audio_in, numpy.ndarray): + self.audio_in = None + self.raw_inputs = audio_in # set the sample_rate of audio_in if checking_audio_fs is valid if checking_audio_fs is not None: @@ -516,23 +505,12 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): logger.info(f"Decoding with {inputs['audio_format']} files ...") data_cmd: Sequence[Tuple[str, str, str]] - if self.cmd['code_base'] == 'funasr': - if isinstance(self.audio_in, bytes): - data_cmd = [self.audio_in, 'speech', 'bytes'] - elif isinstance(self.audio_in, str): - data_cmd = [self.audio_in, 'speech', 'sound'] - elif self.raw_inputs is not None: - data_cmd = None - else: - if inputs['audio_format'] == 'wav' or inputs[ - 'audio_format'] == 'pcm': - data_cmd = ['speech', 'sound'] - elif inputs['audio_format'] == 'kaldi_ark': - data_cmd = ['speech', 'kaldi_ark'] - elif inputs['audio_format'] == 'tfrecord': - data_cmd = ['speech', 'tfrecord'] - if inputs.__contains__('mvn_file'): - data_cmd.append(inputs['mvn_file']) + if isinstance(self.audio_in, bytes): + data_cmd = [self.audio_in, 'speech', 'bytes'] + elif isinstance(self.audio_in, str): + data_cmd = [self.audio_in, 'speech', 'sound'] + elif self.raw_inputs is not None: + data_cmd = None # generate asr inference command self.cmd['name_and_type'] = data_cmd @@ -614,34 +592,9 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): return ref_list def run_inference(self, cmd, **kwargs): - asr_result = [] - if self.framework == Frameworks.torch and cmd['code_base'] == 'funasr': - asr_result = self.funasr_infer_modelscope( - cmd['name_and_type'], cmd['raw_inputs'], cmd['output_dir'], - cmd['fs'], cmd['param_dict'], **kwargs) - - elif self.framework == Frameworks.tf: - from easyasr import asr_inference_paraformer_tf - if hasattr(asr_inference_paraformer_tf, 'set_parameters'): - asr_inference_paraformer_tf.set_parameters( - language=cmd['lang']) - else: - # in order to support easyasr-0.0.2 - cmd['fs'] = cmd['fs']['model_fs'] - - asr_result = asr_inference_paraformer_tf.asr_inference( - ngpu=cmd['ngpu'], - name_and_type=cmd['name_and_type'], - audio_lists=cmd['audio_in'], - idx_text_file=cmd['idx_text'], - asr_model_file=cmd['asr_model_file'], - vocab_file=cmd['vocab_file'], - am_mvn_file=cmd['cmvn_file'], - predictions_file=cmd['predictions_file'], - fs=cmd['fs'], - hop_length=cmd['hop_length'], - feature_dims=cmd['feature_dims'], - sampled_ids=cmd['sampled_ids'], - sampled_lengths=cmd['sampled_lengths']) + asr_result = self.funasr_infer_modelscope(cmd['name_and_type'], + cmd['raw_inputs'], + cmd['output_dir'], cmd['fs'], + cmd['param_dict'], **kwargs) return asr_result diff --git a/modelscope/preprocessors/asr.py b/modelscope/preprocessors/asr.py index ea867775..4696c675 100644 --- a/modelscope/preprocessors/asr.py +++ b/modelscope/preprocessors/asr.py @@ -74,14 +74,6 @@ class WavToScp(Preprocessor): if code_base != 'funasr': cmd = self.config_checking(cmd) cmd = self.env_setting(cmd) - if audio_format == 'wav': - cmd['audio_lists'] = self.scp_generation_from_wav(cmd) - elif audio_format == 'kaldi_ark': - cmd['audio_lists'] = self.scp_generation_from_ark(cmd) - elif audio_format == 'tfrecord': - cmd['audio_lists'] = os.path.join(cmd['wav_path'], 'data.records') - elif audio_format == 'pcm' or audio_format == 'scp': - cmd['audio_lists'] = audio_in return cmd @@ -235,63 +227,4 @@ class WavToScp(Preprocessor): inputs['model_lang'] = inputs['model_config']['lang'] else: inputs['model_lang'] = 'zh-cn' - return inputs - - def scp_generation_from_wav(self, inputs: Dict[str, Any]) -> List[Any]: - """scp generation from waveform files - """ - - # find all waveform files - wav_list = [] - if inputs['recog_type'] == 'wav': - file_path = inputs['wav_path'] - if os.path.isfile(file_path): - if file_path.endswith('.wav') or file_path.endswith('.WAV'): - wav_list.append(file_path) - else: - from easyasr.common import asr_utils - wav_dir: str = inputs['wav_path'] - wav_list = asr_utils.recursion_dir_all_wav(wav_list, wav_dir) - - list_count: int = len(wav_list) - inputs['wav_count'] = list_count - - # store all wav into audio list - audio_lists = [] - j: int = 0 - while j < list_count: - wav_file = wav_list[j] - wave_key: str = os.path.splitext(os.path.basename(wav_file))[0] - item = {'key': wave_key, 'file': wav_file} - audio_lists.append(item) - j += 1 - - return audio_lists - - def scp_generation_from_ark(self, inputs: Dict[str, Any]) -> List[Any]: - """scp generation from kaldi ark file - """ - - ark_scp_path = os.path.join(inputs['wav_path'], 'data.scp') - ark_file_path = os.path.join(inputs['wav_path'], 'data.ark') - assert os.path.exists(ark_scp_path), 'data.scp does not exist' - assert os.path.exists(ark_file_path), 'data.ark does not exist' - - with open(ark_scp_path, 'r', encoding='utf-8') as f: - lines = f.readlines() - - # store all ark item into audio list - audio_lists = [] - for line in lines: - outs = line.strip().split(' ') - if len(outs) == 2: - key = outs[0] - sub = outs[1].split(':') - if len(sub) == 2: - nums = sub[1] - content = ark_file_path + ':' + nums - item = {'key': key, 'file': content} - audio_lists.append(item) - - return audio_lists From f0d519219f02fbb5e786ecc9a3ce2007978364e9 Mon Sep 17 00:00:00 2001 From: "wangmeng.xwm" Date: Fri, 12 May 2023 16:06:52 +0800 Subject: [PATCH 26/76] add clip_interrogator Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12388571 * init commit of clip_interrogator * fix format * fix environment dependencies * fix backend.mps * fix test image path * add model introduction and paper link * add comment example for pipeline, fix device control --- modelscope/metainfo.py | 2 + modelscope/models/multi_modal/__init__.py | 4 +- .../multi_modal/clip_interrogator/__init__.py | 1 + .../multi_modal/clip_interrogator/model.py | 599 ++++++++++++++++++ .../multi_modal/image_captioning_pipeline.py | 23 +- modelscope/preprocessors/__init__.py | 6 +- modelscope/preprocessors/multi_modal.py | 14 + requirements/multi-modal.txt | 3 +- tests/pipelines/test_clip_interrogator.py | 41 ++ 9 files changed, 686 insertions(+), 7 deletions(-) create mode 100644 modelscope/models/multi_modal/clip_interrogator/__init__.py create mode 100644 modelscope/models/multi_modal/clip_interrogator/model.py create mode 100644 tests/pipelines/test_clip_interrogator.py diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index bc3d99b9..f07192ea 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -200,6 +200,7 @@ class Models(object): hitea = 'hitea' soonet = 'soonet' efficient_diffusion_tuning = 'efficient-diffusion-tuning' + clip_interrogator = 'clip-interrogator' # science models unifold = 'unifold' @@ -1026,6 +1027,7 @@ class Preprocessors(object): vldoc_preprocessor = 'vldoc-preprocessor' hitea_tasks_preprocessor = 'hitea-tasks-preprocessor' diffusion_image_generation_preprocessor = 'diffusion-image-generation-preprocessor' + image_captioning_clip_interrogator_preprocessor = 'image-captioning-clip-interrogator-preprocessor' # science preprocessor unifold_preprocessor = 'unifold-preprocessor' diff --git a/modelscope/models/multi_modal/__init__.py b/modelscope/models/multi_modal/__init__.py index e85c48fb..cfcc0361 100644 --- a/modelscope/models/multi_modal/__init__.py +++ b/modelscope/models/multi_modal/__init__.py @@ -20,6 +20,7 @@ if TYPE_CHECKING: from .vldoc import VLDocForDocVLEmbedding from .video_synthesis import TextToVideoSynthesis from .efficient_diffusion_tuning import EfficientStableDiffusion + from .clip_interrogator import CLIP_Interrogator else: _import_structure = { @@ -37,7 +38,8 @@ else: ['MultiStageDiffusionForTextToImageSynthesis'], 'vldoc': ['VLDocForDocVLEmbedding'], 'video_synthesis': ['TextToVideoSynthesis'], - 'efficient_diffusion_tuning': ['EfficientStableDiffusion'] + 'efficient_diffusion_tuning': ['EfficientStableDiffusion'], + 'clip_interrogator': ['CLIP_Interrogator'], } import sys diff --git a/modelscope/models/multi_modal/clip_interrogator/__init__.py b/modelscope/models/multi_modal/clip_interrogator/__init__.py new file mode 100644 index 00000000..96fefbf6 --- /dev/null +++ b/modelscope/models/multi_modal/clip_interrogator/__init__.py @@ -0,0 +1 @@ +from .model import CLIP_Interrogator diff --git a/modelscope/models/multi_modal/clip_interrogator/model.py b/modelscope/models/multi_modal/clip_interrogator/model.py new file mode 100644 index 00000000..a7e27cbd --- /dev/null +++ b/modelscope/models/multi_modal/clip_interrogator/model.py @@ -0,0 +1,599 @@ +# This implementation is adopted from CLIP-Interrogator, made pubicly available under the MIT License at +# https://github.com/pharmapsychotic/clip-interrogator/blob/main/clip_interrogator/clip_interrogator.py + +import hashlib +import math +import os +import time +from dataclasses import dataclass +from typing import List, Optional + +import numpy as np +import open_clip +import requests +import torch +import torchvision.transforms as transforms +from PIL import Image +from safetensors.numpy import load_file, save_file +from tqdm import tqdm +from transformers import (AutoModelForCausalLM, AutoProcessor, + Blip2ForConditionalGeneration, + BlipForConditionalGeneration) + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.preprocessors import LoadImage +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['CLIP_Interrogator'] + +CAPTION_MODELS = { + 'blip-base': 'blip-image-captioning-base', + 'blip-large': 'blip-image-captioning-large', + 'blip2-2.7b': 'blip2-opt-2.7b', + 'blip2-flan-t5-xl': 'blip2-flan-t5-xl', + 'git-large-coco': 'git-large-coco', +} + + +@dataclass +class Config: + # models can optionally be passed in directly + caption_model = None + caption_processor = None + clip_model = None + clip_preprocess = None + + # blip settings + caption_max_length: int = 32 + caption_model_name: Optional[ + str] = 'blip-large' # use a key from CAPTION_MODELS or None + caption_offload: bool = False + + # clip settings + clip_model_name: str = 'ViT-L-14/openai' + clip_model_path: Optional[str] = None + clip_offload: bool = False + + # interrogator settings + cache_path: str = 'cache' # path to store cached text embeddings + download_cache: bool = False # when true, cached embeds are downloaded from huggingface + chunk_size: int = 2048 # batch size for CLIP, use smaller for lower VRAM + data_path: str = os.path.join(os.path.dirname(__file__), 'data') + device: str = ('cuda' if torch.cuda.is_available() else 'cpu') + flavor_intermediate_count: int = 2048 + quiet: bool = False # when quiet progress bars are not shown + + def apply_low_vram_defaults(self): + self.caption_model_name = 'blip-base' + self.caption_offload = True + self.clip_offload = True + self.chunk_size = 1024 + self.flavor_intermediate_count = 1024 + + +# CLIP-Interrogator utilize CLIP and BLIP to generate rich caption for images. +# CLIP is a zero-shot image classifier which can be used to generate image and text embeddings. +# BLIP is a new VLP framework which transfers flexibly to both vision-language understanding and generation tasks. +# BLIP effectively utilizes the noisy web data by bootstrapping the captions, where +# a captioner generates synthetic captions and a filter removes the noisy ones. +# Please infer to the paper CLIP: Learning Transferable Visual Models From Natural Language Supervision +# https://arxiv.org/abs/2103.00020 +# BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation +# https://arxiv.org/abs/2201.12086 + + +class Interrogator(): + + def __init__(self, config: Config): + self.config = config + self.device = config.device + self.dtype = torch.float16 if self.device == 'cuda' else torch.float32 + self.caption_offloaded = True + self.clip_offloaded = True + self.load_caption_model() + self.load_clip_model() + + def load_caption_model(self): + if self.config.caption_model is None and self.config.caption_model_name: + if not self.config.quiet: + print( + f'Loading caption model {self.config.caption_model_name}...' + ) + + model_path = CAPTION_MODELS[self.config.caption_model_name] + if self.config.caption_model_name.startswith('git-'): + caption_model = AutoModelForCausalLM.from_pretrained( + os.path.join(self.config.cache_path, model_path), + torch_dtype=torch.float32) + elif self.config.caption_model_name.startswith('blip2-'): + caption_model = Blip2ForConditionalGeneration.from_pretrained( + os.path.join(self.config.cache_path, model_path), + torch_dtype=self.dtype) + else: + caption_model = BlipForConditionalGeneration.from_pretrained( + os.path.join(self.config.cache_path, model_path), + torch_dtype=self.dtype) + self.caption_processor = AutoProcessor.from_pretrained( + os.path.join(self.config.cache_path, model_path)) + + caption_model.eval() + if not self.config.caption_offload: + caption_model = caption_model.to(self.config.device) + self.caption_model = caption_model + else: + self.caption_model = self.config.caption_model + self.caption_processor = self.config.caption_processor + + def load_clip_model(self): + start_time = time.time() + config = self.config + + clip_model_name, clip_model_pretrained_name = config.clip_model_name.split( + '/', 2) + + if config.clip_model is None: + if not config.quiet: + print(f'Loading CLIP model {config.clip_model_name}...') + + self.clip_model, _, self.clip_preprocess = open_clip.create_model_and_transforms( + clip_model_name, + pretrained=clip_model_pretrained_name, + precision='fp16' if config.device == 'cuda' else 'fp32', + device=config.device, + jit=False, + cache_dir=config.clip_model_path) + self.clip_model.eval() + else: + self.clip_model = config.clip_model + self.clip_preprocess = config.clip_preprocess + self.tokenize = open_clip.get_tokenizer(clip_model_name) + + sites = [ + 'Artstation', 'behance', 'cg society', 'cgsociety', 'deviantart', + 'dribbble', 'flickr', 'instagram', 'pexels', 'pinterest', + 'pixabay', 'pixiv', 'polycount', 'reddit', 'shutterstock', + 'tumblr', 'unsplash', 'zbrush central' + ] + trending_list = [site for site in sites] + trending_list.extend(['trending on ' + site for site in sites]) + trending_list.extend(['featured on ' + site for site in sites]) + trending_list.extend([site + ' contest winner' for site in sites]) + + raw_artists = load_list(config.data_path, 'artists.txt') + artists = [f'by {a}' for a in raw_artists] + artists.extend([f'inspired by {a}' for a in raw_artists]) + + self._prepare_clip() + self.artists = LabelTable(artists, 'artists', self) + self.flavors = LabelTable( + load_list(config.data_path, 'flavors.txt'), 'flavors', self) + self.mediums = LabelTable( + load_list(config.data_path, 'mediums.txt'), 'mediums', self) + self.movements = LabelTable( + load_list(config.data_path, 'movements.txt'), 'movements', self) + self.trendings = LabelTable(trending_list, 'trendings', self) + self.negative = LabelTable( + load_list(config.data_path, 'negative.txt'), 'negative', self) + + end_time = time.time() + if not config.quiet: + print( + f'Loaded CLIP model and data in {end_time-start_time:.2f} seconds.' + ) + + def chain(self, + image_features: torch.Tensor, + phrases: List[str], + best_prompt: str = '', + best_sim: float = 0, + min_count: int = 8, + max_count: int = 32, + desc='Chaining', + reverse: bool = False) -> str: + self._prepare_clip() + + phrases = set(phrases) + if not best_prompt: + best_prompt = self.rank_top( + image_features, [f for f in phrases], reverse=reverse) + best_sim = self.similarity(image_features, best_prompt) + phrases.remove(best_prompt) + curr_prompt, curr_sim = best_prompt, best_sim + + def check(addition: str, idx: int) -> bool: + nonlocal best_prompt, best_sim, curr_prompt, curr_sim + prompt = curr_prompt + ', ' + addition + sim = self.similarity(image_features, prompt) + if reverse: + sim = -sim + + if sim > best_sim: + best_prompt, best_sim = prompt, sim + if sim > curr_sim or idx < min_count: + curr_prompt, curr_sim = prompt, sim + return True + return False + + for idx in tqdm( + range(max_count), desc=desc, disable=self.config.quiet): + best = self.rank_top( + image_features, [f'{curr_prompt}, {f}' for f in phrases], + reverse=reverse) + flave = best[len(curr_prompt) + 2:] + if not check(flave, idx): + break + if _prompt_at_max_len(curr_prompt, self.tokenize): + break + phrases.remove(flave) + + return best_prompt + + def generate_caption(self, pil_image: Image) -> str: + assert self.caption_model is not None, 'No caption model loaded.' + self._prepare_caption() + inputs = self.caption_processor( + images=pil_image, return_tensors='pt').to(self.device) + if not self.config.caption_model_name.startswith('git-'): + inputs = inputs.to(self.dtype) + tokens = self.caption_model.generate( + **inputs, max_new_tokens=self.config.caption_max_length) + return self.caption_processor.batch_decode( + tokens, skip_special_tokens=True)[0].strip() + + def image_to_features(self, image: Image) -> torch.Tensor: + self._prepare_clip() + images = self.clip_preprocess(image).unsqueeze(0).to(self.device) + with torch.no_grad(), torch.cuda.amp.autocast(): + image_features = self.clip_model.encode_image(images) + image_features /= image_features.norm(dim=-1, keepdim=True) + return image_features + + def interrogate_classic(self, + image: Image, + max_flavors: int = 3, + caption: Optional[str] = None) -> str: + """Classic mode creates a prompt in a standard format first describing the image, + then listing the artist, trending, movement, and flavor text modifiers.""" + caption = caption or self.generate_caption(image) + image_features = self.image_to_features(image) + + medium = self.mediums.rank(image_features, 1)[0] + artist = self.artists.rank(image_features, 1)[0] + trending = self.trendings.rank(image_features, 1)[0] + movement = self.movements.rank(image_features, 1)[0] + flaves = ', '.join(self.flavors.rank(image_features, max_flavors)) + + if caption.startswith(medium): + prompt = f'{caption} {artist}, {trending}, {movement}, {flaves}' + else: + prompt = f'{caption}, {medium} {artist}, {trending}, {movement}, {flaves}' + + return _truncate_to_fit(prompt, self.tokenize) + + def interrogate_fast(self, + image: Image, + max_flavors: int = 32, + caption: Optional[str] = None) -> str: + """Fast mode simply adds the top ranked terms after a caption. It generally results in + better similarity between generated prompt and image than classic mode, but the prompts + are less readable.""" + caption = caption or self.generate_caption(image) + image_features = self.image_to_features(image) + merged = _merge_tables([ + self.artists, self.flavors, self.mediums, self.movements, + self.trendings + ], self) + tops = merged.rank(image_features, max_flavors) + return _truncate_to_fit(caption + ', ' + ', '.join(tops), + self.tokenize) + + def interrogate_negative(self, image: Image, max_flavors: int = 32) -> str: + """Negative mode chains together the most dissimilar terms to the image. It can be used + to help build a negative prompt to pair with the regular positive prompt and often + improve the results of generated images particularly with Stable Diffusion 2.""" + image_features = self.image_to_features(image) + flaves = self.flavors.rank( + image_features, + self.config.flavor_intermediate_count, + reverse=True) + flaves = flaves + self.negative.labels + return self.chain( + image_features, + flaves, + max_count=max_flavors, + reverse=True, + desc='Negative chain') + + def interrogate(self, + image: Image, + min_flavors: int = 8, + max_flavors: int = 32, + caption: Optional[str] = None) -> str: + caption = caption or self.generate_caption(image) + image_features = self.image_to_features(image) + + merged = _merge_tables([ + self.artists, self.flavors, self.mediums, self.movements, + self.trendings + ], self) + flaves = merged.rank(image_features, + self.config.flavor_intermediate_count) + best_prompt, best_sim = caption, self.similarity( + image_features, caption) + best_prompt = self.chain( + image_features, + flaves, + best_prompt, + best_sim, + min_count=min_flavors, + max_count=max_flavors, + desc='Flavor chain') + + fast_prompt = self.interrogate_fast( + image, max_flavors, caption=caption) + classic_prompt = self.interrogate_classic( + image, max_flavors, caption=caption) + candidates = [caption, classic_prompt, fast_prompt, best_prompt] + return candidates[np.argmax( + self.similarities(image_features, candidates))] + + def rank_top(self, + image_features: torch.Tensor, + text_array: List[str], + reverse: bool = False) -> str: + self._prepare_clip() + text_tokens = self.tokenize([text + for text in text_array]).to(self.device) + with torch.no_grad(), torch.cuda.amp.autocast(): + text_features = self.clip_model.encode_text(text_tokens) + text_features /= text_features.norm(dim=-1, keepdim=True) + similarity = text_features @ image_features.T + if reverse: + similarity = -similarity + return text_array[similarity.argmax().item()] + + def similarity(self, image_features: torch.Tensor, text: str) -> float: + self._prepare_clip() + text_tokens = self.tokenize([text]).to(self.device) + with torch.no_grad(), torch.cuda.amp.autocast(): + text_features = self.clip_model.encode_text(text_tokens) + text_features /= text_features.norm(dim=-1, keepdim=True) + similarity = text_features @ image_features.T + return similarity[0][0].item() + + def similarities(self, image_features: torch.Tensor, + text_array: List[str]) -> List[float]: + self._prepare_clip() + text_tokens = self.tokenize([text + for text in text_array]).to(self.device) + with torch.no_grad(), torch.cuda.amp.autocast(): + text_features = self.clip_model.encode_text(text_tokens) + text_features /= text_features.norm(dim=-1, keepdim=True) + similarity = text_features @ image_features.T + return similarity.T[0].tolist() + + def _prepare_caption(self): + if self.config.clip_offload and not self.clip_offloaded: + self.clip_model = self.clip_model.to('cpu') + self.clip_offloaded = True + if self.caption_offloaded: + self.caption_model = self.caption_model.to(self.device) + self.caption_offloaded = False + + def _prepare_clip(self): + if self.config.caption_offload and not self.caption_offloaded: + self.caption_model = self.caption_model.to('cpu') + self.caption_offloaded = True + if self.clip_offloaded: + self.clip_model = self.clip_model.to(self.device) + self.clip_offloaded = False + + +class LabelTable(): + + def __init__(self, labels: List[str], desc: str, ci: Interrogator): + clip_model, config = ci.clip_model, ci.config + self.chunk_size = config.chunk_size + self.config = config + self.device = config.device + self.embeds = [] + self.labels = labels + self.tokenize = ci.tokenize + + hash = hashlib.sha256(','.join(labels).encode()).hexdigest() + sanitized_name = self.config.clip_model_name.replace('/', '_').replace( + '@', '_') + self._load_cached(desc, hash, sanitized_name) + + if len(self.labels) != len(self.embeds): + self.embeds = [] + chunks = np.array_split( + self.labels, max(1, + len(self.labels) / config.chunk_size)) + for chunk in tqdm( + chunks, + desc=f'Preprocessing {desc}' if desc else None, + disable=self.config.quiet): + text_tokens = self.tokenize(chunk).to(self.device) + with torch.no_grad(), torch.cuda.amp.autocast(): + text_features = clip_model.encode_text(text_tokens) + text_features /= text_features.norm(dim=-1, keepdim=True) + text_features = text_features.half().cpu().numpy() + for i in range(text_features.shape[0]): + self.embeds.append(text_features[i]) + + if desc and self.config.cache_path: + os.makedirs(self.config.cache_path, exist_ok=True) + cache_filepath = os.path.join( + self.config.cache_path, + f'{sanitized_name}_{desc}.safetensors') + tensors = { + 'embeds': np.stack(self.embeds), + 'hash': np.array([ord(c) for c in hash], dtype=np.int8) + } + save_file(tensors, cache_filepath) + + if self.device == 'cpu' or self.device == torch.device('cpu'): + self.embeds = [e.astype(np.float32) for e in self.embeds] + + def _load_cached(self, desc: str, hash: str, sanitized_name: str) -> bool: + if self.config.cache_path is None or desc is None: + return False + + cached_safetensors = os.path.join( + self.config.cache_path, f'{sanitized_name}_{desc}.safetensors') + + if os.path.exists(cached_safetensors): + try: + tensors = load_file(cached_safetensors) + except Exception as e: + print(f'Failed to load {cached_safetensors}') + print(e) + return False + if 'hash' in tensors and 'embeds' in tensors: + if np.array_equal( + tensors['hash'], + np.array([ord(c) for c in hash], dtype=np.int8)): + self.embeds = tensors['embeds'] + if len(self.embeds.shape) == 2: + self.embeds = [ + self.embeds[i] for i in range(self.embeds.shape[0]) + ] + return True + + return False + + def _rank(self, + image_features: torch.Tensor, + text_embeds: torch.Tensor, + top_count: int = 1, + reverse: bool = False) -> str: + top_count = min(top_count, len(text_embeds)) + text_embeds = torch.stack([torch.from_numpy(t) + for t in text_embeds]).to(self.device) + with torch.cuda.amp.autocast(): + similarity = image_features @ text_embeds.T + if reverse: + similarity = -similarity + _, top_labels = similarity.float().cpu().topk(top_count, dim=-1) + return [top_labels[0][i].numpy() for i in range(top_count)] + + def rank(self, + image_features: torch.Tensor, + top_count: int = 1, + reverse: bool = False) -> List[str]: + if len(self.labels) <= self.chunk_size: + tops = self._rank( + image_features, + self.embeds, + top_count=top_count, + reverse=reverse) + return [self.labels[i] for i in tops] + + num_chunks = int(math.ceil(len(self.labels) / self.chunk_size)) + keep_per_chunk = int(self.chunk_size / num_chunks) + + top_labels, top_embeds = [], [] + for chunk_idx in tqdm(range(num_chunks), disable=self.config.quiet): + start = chunk_idx * self.chunk_size + stop = min(start + self.chunk_size, len(self.embeds)) + tops = self._rank( + image_features, + self.embeds[start:stop], + top_count=keep_per_chunk, + reverse=reverse) + top_labels.extend([self.labels[start + i] for i in tops]) + top_embeds.extend([self.embeds[start + i] for i in tops]) + + tops = self._rank(image_features, top_embeds, top_count=top_count) + return [top_labels[i] for i in tops] + + +def _download_file(url: str, + filepath: str, + chunk_size: int = 4 * 1024 * 1024, + quiet: bool = False): + r = requests.get(url, stream=True) + if r.status_code != 200: + return + + file_size = int(r.headers.get('Content-Length', 0)) + filename = url.split('/')[-1] + progress = tqdm( + total=file_size, + unit='B', + unit_scale=True, + desc=filename, + disable=quiet) + with open(filepath, 'wb') as f: + for chunk in r.iter_content(chunk_size=chunk_size): + if chunk: + f.write(chunk) + progress.update(len(chunk)) + progress.close() + + +def _merge_tables(tables: List[LabelTable], ci: Interrogator) -> LabelTable: + m = LabelTable([], None, ci) + for table in tables: + m.labels.extend(table.labels) + m.embeds.extend(table.embeds) + return m + + +def _prompt_at_max_len(text: str, tokenize) -> bool: + tokens = tokenize([text]) + return tokens[0][-1] != 0 + + +def _truncate_to_fit(text: str, tokenize) -> str: + parts = text.split(', ') + new_text = parts[0] + for part in parts[1:]: + if _prompt_at_max_len(new_text + part, tokenize): + break + new_text += ', ' + part + return new_text + + +def list_caption_models() -> List[str]: + return list(CAPTION_MODELS.keys()) + + +def list_clip_models() -> List[str]: + return ['/'.join(x) for x in open_clip.list_pretrained()] + + +def load_list(data_path: str, filename: Optional[str] = None) -> List[str]: + """Load a list of strings from a file.""" + if filename is not None: + data_path = os.path.join(data_path, filename) + with open(data_path, 'r', encoding='utf-8', errors='replace') as f: + items = [line.strip() for line in f.readlines()] + return items + + +@MODELS.register_module( + Tasks.image_captioning, module_name=Models.clip_interrogator) +class CLIP_Interrogator(TorchModel): + + def __init__(self, model_dir, device='cuda', device_id=0, *args, **kwargs): + super().__init__( + model_dir=model_dir, device_id=device_id, *args, **kwargs) + self.device = device + self.dtype = torch.float16 if self.device == 'cuda' else torch.float32 + cf = Config(clip_model_name='ViT-L-14/openai') + cf.data_path = os.path.join(model_dir, 'data') + cf.clip_model_path = model_dir + cf.cache_path = model_dir + self.ci = Interrogator(cf) + + def forward(self, inputs): + image = transforms.ToPILImage()(inputs) + return {'caption': self.ci.interrogate(image)} diff --git a/modelscope/pipelines/multi_modal/image_captioning_pipeline.py b/modelscope/pipelines/multi_modal/image_captioning_pipeline.py index fbab88fd..17b850da 100644 --- a/modelscope/pipelines/multi_modal/image_captioning_pipeline.py +++ b/modelscope/pipelines/multi_modal/image_captioning_pipeline.py @@ -1,15 +1,18 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from typing import Any, Dict, Optional, Union +import numpy as np import torch from modelscope.metainfo import Pipelines -from modelscope.models.multi_modal import MPlugForAllTasks, OfaForAllTasks +from modelscope.models.multi_modal import (CLIP_Interrogator, MPlugForAllTasks, + OfaForAllTasks) from modelscope.pipelines.base import Model, Pipeline from modelscope.pipelines.builder import PIPELINES from modelscope.pipelines.util import batch_process -from modelscope.preprocessors import (MPlugPreprocessor, OfaPreprocessor, - Preprocessor) +from modelscope.preprocessors import ( + ImageCaptioningClipInterrogatorPreprocessor, MPlugPreprocessor, + OfaPreprocessor, Preprocessor, load_image) from modelscope.utils.constant import ModelFile, Tasks from modelscope.utils.logger import get_logger @@ -28,6 +31,17 @@ class ImageCaptioningPipeline(Pipeline): use `model` and `preprocessor` to create a image captioning pipeline for prediction Args: model: model id on modelscope hub. + Examples: + from modelscope.pipelines import pipeline + from modelscope.utils.constant import Tasks + + model_id = 'damo/cv_clip-interrogator' + input_image = "test.png" + + pipeline_ci = pipeline(Tasks.image_captioning, model=model_id) + print(pipeline_ci(input_image)) + + """ super().__init__(model=model, preprocessor=preprocessor, **kwargs) self.model.eval() @@ -39,6 +53,9 @@ class ImageCaptioningPipeline(Pipeline): self.preprocessor = OfaPreprocessor(self.model.model_dir) elif isinstance(self.model, MPlugForAllTasks): self.preprocessor = MPlugPreprocessor(self.model.model_dir) + elif isinstance(self.model, CLIP_Interrogator): + self.preprocessor = ImageCaptioningClipInterrogatorPreprocessor( + ) def _batch(self, data): if isinstance(self.model, OfaForAllTasks): diff --git a/modelscope/preprocessors/__init__.py b/modelscope/preprocessors/__init__.py index 8d74f521..fab055db 100644 --- a/modelscope/preprocessors/__init__.py +++ b/modelscope/preprocessors/__init__.py @@ -20,7 +20,8 @@ if TYPE_CHECKING: from .tts import KanttsDataPreprocessor from .multi_modal import (DiffusionImageGenerationPreprocessor, OfaPreprocessor, MPlugPreprocessor, - HiTeAPreprocessor) + HiTeAPreprocessor, + ImageCaptioningClipInterrogatorPreprocessor) from .nlp import ( DocumentSegmentationTransformersPreprocessor, FaqQuestionAnsweringTransformersPreprocessor, @@ -70,7 +71,8 @@ else: 'tts': ['KanttsDataPreprocessor'], 'multi_modal': [ 'DiffusionImageGenerationPreprocessor', 'OfaPreprocessor', - 'MPlugPreprocessor', 'HiTeAPreprocessor' + 'MPlugPreprocessor', 'HiTeAPreprocessor', + 'ImageCaptioningClipInterrogatorPreprocessor' ], 'nlp': [ 'DocumentSegmentationTransformersPreprocessor', diff --git a/modelscope/preprocessors/multi_modal.py b/modelscope/preprocessors/multi_modal.py index bd37c620..eb7ae339 100644 --- a/modelscope/preprocessors/multi_modal.py +++ b/modelscope/preprocessors/multi_modal.py @@ -642,3 +642,17 @@ class HiTeAPreprocessor(Preprocessor): 'answer_attention_mask': answer.attention_mask.squeeze(), } return output + + +@PREPROCESSORS.register_module( + Fields.multi_modal, + module_name=Preprocessors.image_captioning_clip_interrogator_preprocessor) +class ImageCaptioningClipInterrogatorPreprocessor(Preprocessor): + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def __call__(self, data) -> Dict[str, Any]: + image = load_image(data) + data = np.array(image).transpose(2, 0, 1) + return data diff --git a/requirements/multi-modal.txt b/requirements/multi-modal.txt index 3d75f9b6..b07b99a2 100644 --- a/requirements/multi-modal.txt +++ b/requirements/multi-modal.txt @@ -12,13 +12,14 @@ rapidfuzz # which introduced compatability issues that are being investigated rouge_score<=0.0.4 sacrebleu +safetensors # scikit-video soundfile taming-transformers-rom1504 timm tokenizers torchvision -transformers>=4.12.0 +transformers>=4.27.1 # triton==2.0.0.dev20221120 unicodedata2 zhconv diff --git a/tests/pipelines/test_clip_interrogator.py b/tests/pipelines/test_clip_interrogator.py new file mode 100644 index 00000000..960db86d --- /dev/null +++ b/tests/pipelines/test_clip_interrogator.py @@ -0,0 +1,41 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import unittest + +from PIL import Image + +from modelscope.models import Model +from modelscope.outputs import OutputKeys +from modelscope.pipelines import pipeline +from modelscope.utils.constant import Tasks +from modelscope.utils.demo_utils import DemoCompatibilityCheck +from modelscope.utils.test_utils import test_level + + +class CLIPInterrogatorTest(unittest.TestCase, DemoCompatibilityCheck): + + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') + def test_run_with_image_captioning_with_model(self): + model = Model.from_pretrained('damo/cv_clip-interrogator') + pipeline_caption = pipeline( + task=Tasks.image_captioning, + model=model, + ) + image = 'data/test/images/image_mplug_vqa.jpg' + result = pipeline_caption(image) + print(result[OutputKeys.CAPTION]) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_image_captioning_with_name(self): + pipeline_caption = pipeline( + Tasks.image_captioning, model='damo/cv_clip-interrogator') + image = 'data/test/images/image_mplug_vqa.jpg' + result = pipeline_caption(image) + print(result[OutputKeys.CAPTION]) + + @unittest.skip('demo compatibility test is only enabled on a needed-basis') + def test_demo_compatibility(self): + self.compatibility_check() + + +if __name__ == '__main__': + unittest.main() From cafc90da21fced9bf0b12e76824542c762ea4e92 Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Fri, 12 May 2023 17:25:32 +0800 Subject: [PATCH 27/76] Fix master UT on level0 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12599393 * reset ut level for test_run_with_model_name_for_unite_large and test_run_with_model_name_for_unite_base * fix master UT for level0 * update EasyCVFace2DKeypointsPipelineTest skip info --- tests/pipelines/test_face_2d_keypoints.py | 2 +- tests/pipelines/test_face_reconstruction.py | 2 +- tests/pipelines/test_hand_2d_keypoints.py | 2 +- tests/pipelines/test_hand_detection.py | 2 +- tests/pipelines/test_human_wholebody_keypoint.py | 2 +- tests/pipelines/test_image_panoptic_segmentation.py | 4 ++-- tests/pipelines/test_object_detection.py | 4 ++-- tests/pipelines/test_realtime_object_detection.py | 4 ++-- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/pipelines/test_face_2d_keypoints.py b/tests/pipelines/test_face_2d_keypoints.py index 7ccc8a59..875a0e11 100644 --- a/tests/pipelines/test_face_2d_keypoints.py +++ b/tests/pipelines/test_face_2d_keypoints.py @@ -10,7 +10,7 @@ from modelscope.utils.test_utils import test_level class EasyCVFace2DKeypointsPipelineTest(unittest.TestCase): - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skip('skip easycv related cases') def test_face_2d_keypoints(self): img_path = 'data/test/images/face_detection.png' model_id = 'damo/cv_mobilenet_face-2d-keypoints_alignment' diff --git a/tests/pipelines/test_face_reconstruction.py b/tests/pipelines/test_face_reconstruction.py index b35482fb..c1de97c0 100644 --- a/tests/pipelines/test_face_reconstruction.py +++ b/tests/pipelines/test_face_reconstruction.py @@ -60,7 +60,7 @@ class FaceReconstructionTest(unittest.TestCase, DemoCompatibilityCheck): Tasks.face_reconstruction, model=model_dir) self.pipeline_inference(face_reconstruction, self.test_image) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_run_modelhub(self): face_reconstruction = pipeline( Tasks.face_reconstruction, diff --git a/tests/pipelines/test_hand_2d_keypoints.py b/tests/pipelines/test_hand_2d_keypoints.py index 43b569d0..a243a478 100644 --- a/tests/pipelines/test_hand_2d_keypoints.py +++ b/tests/pipelines/test_hand_2d_keypoints.py @@ -23,7 +23,7 @@ class Hand2DKeypointsPipelineTest(unittest.TestCase): self.assertEqual(results[OutputKeys.KEYPOINTS].shape[2], 3) self.assertEqual(results[OutputKeys.BOXES].shape[1], 4) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skip('skip test in current test level: no pipeline implemented') def test_hand_2d_keypoints_with_default_model(self): img_path = 'data/test/images/hand_keypoints.jpg' diff --git a/tests/pipelines/test_hand_detection.py b/tests/pipelines/test_hand_detection.py index 8a6bbd5a..60366287 100644 --- a/tests/pipelines/test_hand_detection.py +++ b/tests/pipelines/test_hand_detection.py @@ -13,7 +13,7 @@ class ObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): self.task = Tasks.domain_specific_object_detection self.model_id = 'damo/cv_yolox-pai_hand-detection' - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skip('skip test in current test level: no pipeline implemented') def test_hand_detection_pipeline(self): test_image = 'data/test/images/hand_detection.jpg' diff --git a/tests/pipelines/test_human_wholebody_keypoint.py b/tests/pipelines/test_human_wholebody_keypoint.py index 7c5946cc..e0052f77 100644 --- a/tests/pipelines/test_human_wholebody_keypoint.py +++ b/tests/pipelines/test_human_wholebody_keypoint.py @@ -11,7 +11,7 @@ from modelscope.utils.test_utils import test_level class EasyCVFace2DKeypointsPipelineTest(unittest.TestCase): - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skip('skip easycv related cases') def test_human_wholebody_keypoint(self): img_path = 'data/test/images/keypoints_detect/img_test_wholebody.jpg' model_id = 'damo/cv_hrnetw48_human-wholebody-keypoint_image' diff --git a/tests/pipelines/test_image_panoptic_segmentation.py b/tests/pipelines/test_image_panoptic_segmentation.py index 4f12e6af..ccd9eafd 100644 --- a/tests/pipelines/test_image_panoptic_segmentation.py +++ b/tests/pipelines/test_image_panoptic_segmentation.py @@ -19,7 +19,7 @@ class ImagePanopticSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): self.task = Tasks.image_segmentation self.model_id = 'damo/cv_swinL_panoptic-segmentation_cocopan' - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skip('skip test in current test level: no pipeline implemented') def test_image_panoptic_segmentation(self): input_location = 'data/test/images/image_panoptic_segmentation.jpg' pan_segmentor = pipeline(Tasks.image_segmentation, model=self.model_id) @@ -29,7 +29,7 @@ class ImagePanopticSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): cv2.imwrite('result.jpg', draw_img) print('print test_image_panoptic_segmentation return success') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skip('skip test in current test level: no pipeline implemented') def test_image_panoptic_segmentation_from_PIL(self): input_location = 'data/test/images/image_panoptic_segmentation.jpg' pan_segmentor = pipeline(Tasks.image_segmentation, model=self.model_id) diff --git a/tests/pipelines/test_object_detection.py b/tests/pipelines/test_object_detection.py index e4bf6b54..7be6b0f3 100644 --- a/tests/pipelines/test_object_detection.py +++ b/tests/pipelines/test_object_detection.py @@ -47,7 +47,7 @@ class ObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): def test_demo_compatibility(self): self.compatibility_check() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skip('skip test in current test level: no pipeline implemented') def test_image_object_detection_auto_pipeline(self): model_id = 'damo/cv_yolox_image-object-detection-auto' test_image = 'data/test/images/auto_demo.jpg' @@ -59,7 +59,7 @@ class ObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): image_object_detection_auto.show_result(test_image, result, 'auto_demo_ret.jpg') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skip('skip test in current test level: no pipeline implemented') def test_image_object_detection_dino_pipeline(self): model_id = 'damo/cv_swinl_image-object-detection_dino' test_image = 'data/test/images/image_detection.jpg' diff --git a/tests/pipelines/test_realtime_object_detection.py b/tests/pipelines/test_realtime_object_detection.py index 498c09d8..76e907a7 100644 --- a/tests/pipelines/test_realtime_object_detection.py +++ b/tests/pipelines/test_realtime_object_detection.py @@ -22,7 +22,7 @@ class RealtimeObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): self.test_image = 'data/test/images/keypoints_detect/000000438862.jpg' self.task = Tasks.image_object_detection - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skip('skip test in current test level: no pipeline implemented') def test_run_easycv_yolox(self): realtime_object_detection = pipeline( Tasks.image_object_detection, model=self.easycv_small_model_id) @@ -34,7 +34,7 @@ class RealtimeObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): else: raise ValueError('process error') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skip('skip test in current test level: no pipeline implemented') def test_run_easycv_yolox_nano(self): realtime_object_detection = pipeline( Tasks.image_object_detection, model=self.easycv_nano_model_id) From e6306215992944ddbc8a06396a36a6113afef068 Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Fri, 12 May 2023 17:27:19 +0800 Subject: [PATCH 28/76] Virgo SDK supports odps data source 1. Support ODPS datasource for virgo sdk. 2. Adapt "inner_url" parser for single-modal and multi-modal datasets. Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12583695 * add virgo sdk odps data * refine virgo sdk args and odps fetch batch data pipeline * add inner import for odps * del import VirgoDataset in MsDataset * fix import VirgoDataset * support inner url downloading * refine dataset.py and maxcompute_utils.py * add ut for virgo odps data batch * reset unifold ut level as 1 * refine virgo batch data --- .../context/dataset_context_config.py | 9 - .../msdatasets/data_loader/data_loader.py | 76 ++++++--- modelscope/msdatasets/dataset_cls/__init__.py | 2 +- modelscope/msdatasets/dataset_cls/dataset.py | 107 ++++++++++-- .../msdatasets/download/dataset_builder.py | 4 +- .../msdatasets/meta/data_meta_manager.py | 2 +- modelscope/msdatasets/ms_dataset.py | 11 +- .../msdatasets/utils/maxcompute_utils.py | 160 ++++++++++++++++++ modelscope/utils/constant.py | 14 ++ tests/msdatasets/test_ms_dataset.py | 4 +- tests/msdatasets/test_virgo_dataset.py | 22 ++- 11 files changed, 345 insertions(+), 66 deletions(-) create mode 100644 modelscope/msdatasets/utils/maxcompute_utils.py diff --git a/modelscope/msdatasets/context/dataset_context_config.py b/modelscope/msdatasets/context/dataset_context_config.py index 929e68d0..4007d60c 100644 --- a/modelscope/msdatasets/context/dataset_context_config.py +++ b/modelscope/msdatasets/context/dataset_context_config.py @@ -24,7 +24,6 @@ class DatasetContextConfig: self._config_kwargs = kwargs self._dataset_version_cache_root_dir = None self._auth_config = None - self._ext_config: dict = {} # The lock file path for meta-files and data-files self._global_meta_lock_file_path = None @@ -100,11 +99,3 @@ class DatasetContextConfig: @auth_config.setter def auth_config(self, val: BaseAuthConfig): self._auth_config = val - - @property - def ext_config(self) -> dict: - return self._ext_config - - @ext_config.setter - def ext_config(self, val: dict): - self._ext_config = val diff --git a/modelscope/msdatasets/data_loader/data_loader.py b/modelscope/msdatasets/data_loader/data_loader.py index ebda87e6..b1450c61 100644 --- a/modelscope/msdatasets/data_loader/data_loader.py +++ b/modelscope/msdatasets/data_loader/data_loader.py @@ -14,7 +14,7 @@ from modelscope.msdatasets.context.dataset_context_config import \ DatasetContextConfig from modelscope.msdatasets.data_files.data_files_manager import \ DataFilesManager -from modelscope.msdatasets.dataset_cls.dataset import ExternalDataset +from modelscope.msdatasets.dataset_cls import ExternalDataset from modelscope.msdatasets.meta.data_meta_manager import DataMetaManager from modelscope.utils.constant import (DatasetFormations, DatasetPathName, DownloadMode, VirgoDatasetConfig) @@ -191,12 +191,14 @@ class VirgoDownloader(BaseDownloader): """ Fetch virgo meta and build virgo dataset. """ - from modelscope.msdatasets.dataset_cls import VirgoDataset + from modelscope.msdatasets.dataset_cls.dataset import VirgoDataset + import pandas as pd meta_manager = DataMetaManager(self.dataset_context_config) meta_manager.fetch_virgo_meta() self.dataset_context_config = meta_manager.dataset_context_config - self.dataset = VirgoDataset(**self.dataset_context_config.ext_config) + self.dataset = VirgoDataset( + **self.dataset_context_config.config_kwargs) virgo_cache_dir = os.path.join( self.dataset_context_config.cache_root_dir, @@ -209,18 +211,24 @@ class VirgoDownloader(BaseDownloader): meta_content_cache_file = os.path.join(virgo_cache_dir, DatasetPathName.META_NAME, 'meta_content.csv') - meta_content_df = self.dataset.meta - meta_content_df.to_csv(meta_content_cache_file, index=False) - self.dataset.meta_content_cache_file = meta_content_cache_file - self.dataset.virgo_cache_dir = virgo_cache_dir - logger.info(f'Virgo meta content saved to {meta_content_cache_file}') + + if isinstance(self.dataset.meta, pd.DataFrame): + meta_content_df = self.dataset.meta + meta_content_df.to_csv(meta_content_cache_file, index=False) + self.dataset.meta_content_cache_file = meta_content_cache_file + self.dataset.virgo_cache_dir = virgo_cache_dir + logger.info( + f'Virgo meta content saved to {meta_content_cache_file}') def _prepare_and_download(self): """ Fetch data-files from oss-urls in the virgo meta content. """ - if self.dataset_context_config.download_virgo_files: + download_virgo_files = self.dataset_context_config.config_kwargs.pop( + 'download_virgo_files', '') + + if self.dataset.data_type == 0 and download_virgo_files: import requests import json import shutil @@ -228,28 +236,44 @@ class VirgoDownloader(BaseDownloader): from functools import partial def download_file(meta_info_val, data_dir): - file_url = '' + file_url_list = [] + file_path_list = [] try: - file_url = json.loads(meta_info_val)['url'] - is_url = valid_url(file_url) - if is_url: - url_parse_res = urlparse(file_url) - file_name = os.path.basename(url_parse_res.path) + meta_info_val = json.loads(meta_info_val) + # get url first, if not exist, try to get inner_url + file_url = meta_info_val.get('url', '') + if file_url: + file_url_list.append(file_url) else: - raise ValueError(f'Unsupported url: {file_url}') - file_path = os.path.join(data_dir, file_name) + tmp_inner_member_list = meta_info_val.get( + 'inner_url', '') + for item in tmp_inner_member_list: + file_url = item.get('url', '') + if file_url: + file_url_list.append(file_url) + + for one_file_url in file_url_list: + is_url = valid_url(one_file_url) + if is_url: + url_parse_res = urlparse(file_url) + file_name = os.path.basename(url_parse_res.path) + else: + raise ValueError(f'Unsupported url: {file_url}') + file_path = os.path.join(data_dir, file_name) + file_path_list.append((one_file_url, file_path)) + except Exception as e: - logger.error(e) - file_path = '' + logger.error(f'parse virgo meta info error: {e}') + file_path_list = [] - if file_path and not os.path.exists(file_path): - logger.info( - f'Downloading file from {file_url} to {file_path}') - os.makedirs(data_dir, exist_ok=True) - with open(file_path, 'wb') as f: - f.write(requests.get(file_url).content) + for file_url_item, file_path_item in file_path_list: + if file_path_item and not os.path.exists(file_path_item): + logger.info(f'Downloading file to {file_path_item}') + os.makedirs(data_dir, exist_ok=True) + with open(file_path_item, 'wb') as f: + f.write(requests.get(file_url_item).content) - return file_path + return file_path_list self.dataset.download_virgo_files = True download_mode = self.dataset_context_config.download_mode diff --git a/modelscope/msdatasets/dataset_cls/__init__.py b/modelscope/msdatasets/dataset_cls/__init__.py index ced94f2f..a5b2e73d 100644 --- a/modelscope/msdatasets/dataset_cls/__init__.py +++ b/modelscope/msdatasets/dataset_cls/__init__.py @@ -1,3 +1,3 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -from .dataset import ExternalDataset, NativeIterableDataset, VirgoDataset +from .dataset import ExternalDataset, NativeIterableDataset diff --git a/modelscope/msdatasets/dataset_cls/dataset.py b/modelscope/msdatasets/dataset_cls/dataset.py index e5e9664a..9114285e 100644 --- a/modelscope/msdatasets/dataset_cls/dataset.py +++ b/modelscope/msdatasets/dataset_cls/dataset.py @@ -6,9 +6,11 @@ import os import datasets import pandas as pd from datasets import IterableDataset -from PIL import Image -from modelscope.utils.constant import EXTENSIONS_TO_LOAD, VirgoDatasetConfig +from modelscope.msdatasets.utils.maxcompute_utils import MaxComputeUtil +from modelscope.utils.constant import (DEFAULT_MAXCOMPUTE_ENDPOINT, + EXTENSIONS_TO_LOAD, MaxComputeEnvs, + VirgoDatasetConfig) from modelscope.utils.logger import get_logger from modelscope.utils.url_utils import fetch_csv_with_url, valid_url @@ -99,6 +101,7 @@ class NativeIterableDataset(IterableDataset): ex_cache_path = dl_manager.download_and_extract(v) ret[k] = ex_cache_path if k.endswith('Image:FILE'): + from PIL import Image ret[k + ':Object'] = Image.open(fp=ex_cache_path) if k.endswith('Audio:FILE'): import torchaudio @@ -120,7 +123,7 @@ class VirgoDataset(object): _data_type (int): Virgo dataset type, 0-Standard virgo dataset; Others-User define dataset (to be supported) Examples: - >>> from modelscope.msdatasets.dataset_cls import VirgoDataset + >>> from modelscope.msdatasets.dataset_cls.dataset import VirgoDataset >>> input_kwargs = {'metaContent': 'http://xxx-xxx/xxx.csv', 'samplingType': 0} >>> virgo_dataset = VirgoDataset(**input_kwargs) >>> print(virgo_dataset[1]) @@ -138,13 +141,18 @@ class VirgoDataset(object): def __init__(self, **kwargs): self._meta_content: str = '' - self._data_type: int = 0 + self.data_type: int = 0 + self.odps_table_name: str = '' + self.odps_table_partition: str = None + self._odps_utils: MaxComputeUtil = None + self.config_kwargs = kwargs + self._meta: pd.DataFrame = pd.DataFrame() - if VirgoDatasetConfig.meta_content in kwargs: - self._meta_content = kwargs.pop(VirgoDatasetConfig.meta_content) - if VirgoDatasetConfig.sampling_type in kwargs: - self._data_type = kwargs.pop(VirgoDatasetConfig.sampling_type) + self._meta_content = self.config_kwargs.pop( + VirgoDatasetConfig.meta_content, '') + self.data_type = self.config_kwargs.pop( + VirgoDatasetConfig.sampling_type, 0) self._check_variables() self._parse_meta() @@ -153,15 +161,46 @@ class VirgoDataset(object): self.virgo_cache_dir = '' self.download_virgo_files: bool = False + self.odps_table_ins = None + self.odps_reader_ins = None + self.odps_batch_size = self.config_kwargs.pop('odps_batch_size', 100) + self.odps_limit = self.config_kwargs.pop('odps_limit', None) + self.odps_drop_last = self.config_kwargs.pop('odps_drop_last', False) + if self._odps_utils: + self.odps_table_ins, self.odps_reader_ins = self._odps_utils.get_table_reader_ins( + self.odps_table_name, self.odps_table_partition) + def __getitem__(self, index): + if self.odps_reader_ins: + return MaxComputeUtil.gen_reader_item( + reader=self.odps_reader_ins, + index=index, + batch_size_in=self.odps_batch_size, + limit_in=self.odps_limit, + drop_last_in=self.odps_drop_last, + partitions=self.odps_table_ins.table_schema.partitions, + columns=self.odps_table_ins.table_schema.names) return self._meta.iloc[index].to_dict() def __len__(self): + if isinstance(self._meta, dict): + return self._meta.get('odpsCount', 0) return len(self._meta) def __iter__(self): - for _, row in self._meta.iterrows(): - yield row.to_dict() + if self.odps_reader_ins: + odps_batch_data = MaxComputeUtil.gen_reader_batch( + reader=self.odps_reader_ins, + batch_size_in=self.odps_batch_size, + limit_in=self.odps_limit, + drop_last_in=self.odps_drop_last, + partitions=self.odps_table_ins.table_schema.partitions, + columns=self.odps_table_ins.table_schema.names) + for batch in odps_batch_data: + yield batch + else: + for _, row in self._meta.iterrows(): + yield row.to_dict() @property def meta(self) -> pd.DataFrame: @@ -173,8 +212,44 @@ class VirgoDataset(object): def _parse_meta(self): # Fetch csv content - meta_content_df = fetch_csv_with_url(self._meta_content) - self._meta = meta_content_df + if isinstance(self._meta_content, str) and valid_url( + self._meta_content): + meta_content_df = fetch_csv_with_url(self._meta_content) + self._meta = meta_content_df + elif isinstance(self._meta_content, dict): + self._meta = self._meta_content + self.odps_table_name = self._meta.get('odpsTableName', '') + self.odps_table_partition = self._meta.get('odpsTablePartition', + None) + self._odps_utils = self._get_odps_info() + else: + raise 'The meta content must be url or dict.' + + @staticmethod + def _get_odps_info() -> MaxComputeUtil: + """ + Get MaxComputeUtil instance. + + Args: + None + + Returns: + MaxComputeUtil instance. + """ + access_id = os.environ.get(MaxComputeEnvs.ACCESS_ID, '') + access_key = os.environ.get(MaxComputeEnvs.ACCESS_SECRET_KEY, '') + proj_name = os.environ.get(MaxComputeEnvs.PROJECT_NAME, '') + endpoint = os.environ.get(MaxComputeEnvs.ENDPOINT, + DEFAULT_MAXCOMPUTE_ENDPOINT) + + if not access_id or not access_key or not proj_name: + raise ValueError( + f'Please set MaxCompute envs for Virgo: {MaxComputeEnvs.ACCESS_ID}, ' + f'{MaxComputeEnvs.ACCESS_SECRET_KEY}, {MaxComputeEnvs.PROJECT_NAME}, ' + f'{MaxComputeEnvs.ENDPOINT}(default: http://service-corp.odps.aliyun-inc.com/api)' + ) + + return MaxComputeUtil(access_id, access_key, proj_name, endpoint) def _check_variables(self): """Check member variables in this class. @@ -183,7 +258,9 @@ class VirgoDataset(object): """ if not self._meta_content: raise 'Them meta content cannot be empty.' - if self._data_type != 0: - raise 'Supported samplingType should be 0, others are not supported yet.' - if not valid_url(self._meta_content): + if self.data_type not in [0, 1]: + raise 'Supported samplingType should be 0 or 1, others are not supported yet.' + if self.data_type == 0 and not valid_url(self._meta_content): raise 'The meta content must be url when data type is 0.' + if self.data_type == 1 and not isinstance(self._meta_content, dict): + raise 'The meta content must be dict when data type is 1.' diff --git a/modelscope/msdatasets/download/dataset_builder.py b/modelscope/msdatasets/download/dataset_builder.py index 73a3a1a1..8ad5243a 100644 --- a/modelscope/msdatasets/download/dataset_builder.py +++ b/modelscope/msdatasets/download/dataset_builder.py @@ -18,8 +18,8 @@ from datasets.utils.py_utils import map_nested from modelscope.hub.api import HubApi from modelscope.msdatasets.context.dataset_context_config import \ DatasetContextConfig -from modelscope.msdatasets.dataset_cls.dataset import (ExternalDataset, - NativeIterableDataset) +from modelscope.msdatasets.dataset_cls import (ExternalDataset, + NativeIterableDataset) from modelscope.msdatasets.download.download_manager import \ DataStreamingDownloadManager from modelscope.msdatasets.utils.dataset_utils import \ diff --git a/modelscope/msdatasets/meta/data_meta_manager.py b/modelscope/msdatasets/meta/data_meta_manager.py index 6a34d5d6..0fa74c37 100644 --- a/modelscope/msdatasets/meta/data_meta_manager.py +++ b/modelscope/msdatasets/meta/data_meta_manager.py @@ -146,7 +146,7 @@ class DataMetaManager(object): meta_content = self.api.get_virgo_meta( dataset_id=virgo_dataset_id, version=version) - self.dataset_context_config.ext_config = meta_content + self.dataset_context_config.config_kwargs.update(meta_content) def _fetch_meta_from_cache(self, meta_cache_dir): local_paths = defaultdict(list) diff --git a/modelscope/msdatasets/ms_dataset.py b/modelscope/msdatasets/ms_dataset.py index 6729e4e6..912e061d 100644 --- a/modelscope/msdatasets/ms_dataset.py +++ b/modelscope/msdatasets/ms_dataset.py @@ -18,8 +18,7 @@ from modelscope.msdatasets.data_loader.data_loader_manager import ( LocalDataLoaderManager, LocalDataLoaderType, RemoteDataLoaderManager, RemoteDataLoaderType) from modelscope.msdatasets.dataset_cls import (ExternalDataset, - NativeIterableDataset, - VirgoDataset) + NativeIterableDataset) from modelscope.msdatasets.dataset_cls.custom_datasets.builder import \ build_custom_dataset from modelscope.msdatasets.utils.delete_utils import DatasetDeleteManager @@ -172,7 +171,7 @@ class MsDataset: use_streaming: Optional[bool] = False, custom_cfg: Optional[Config] = Config(), **config_kwargs, - ) -> Union[dict, 'MsDataset', NativeIterableDataset, VirgoDataset]: + ) -> Union[dict, 'MsDataset', NativeIterableDataset]: """Load a MsDataset from the ModelScope Hub, Hugging Face Hub, urls, or a local dataset. Args: @@ -190,9 +189,6 @@ class MsDataset: data_files (str or Sequence or Mapping, optional): Path(s) to source data file(s). split (str, optional): Which split of the data to load. hub (Hubs or str, optional): When loading from a remote hub, where it is from. default Hubs.modelscope - download_mode (DownloadMode or str, optional): - How to treat existing datasets. default DownloadMode.REUSE_DATASET_IF_EXISTS - config_kwargs (additional keyword arguments): Keyword arguments to be passed download_mode (DownloadMode or str, optional): How to treat existing datasets. default DownloadMode.REUSE_DATASET_IF_EXISTS cache_dir (str, Optional): User-define local cache directory. @@ -300,9 +296,6 @@ class MsDataset: cache_dir = os.path.join(CACHE_HOME, 'virgo', 'hub', 'datasets') dataset_context_config.cache_root_dir = cache_dir - if 'download_virgo_files' in config_kwargs: - dataset_context_config.download_virgo_files = config_kwargs.pop( - 'download_virgo_files') virgo_downloader = VirgoDownloader(dataset_context_config) virgo_downloader.process() diff --git a/modelscope/msdatasets/utils/maxcompute_utils.py b/modelscope/msdatasets/utils/maxcompute_utils.py new file mode 100644 index 00000000..83c6370d --- /dev/null +++ b/modelscope/msdatasets/utils/maxcompute_utils.py @@ -0,0 +1,160 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math + +import pandas as pd + + +class MaxComputeUtil: + """ + MaxCompute util class. + + Args: + access_id: your access id of MaxCompute + access_key: access key of MaxCompute + project_name: your project name of MaxCompute + endpoint: endpoint of MaxCompute + + Attributes: + _odps: ODPS object + + """ + + def __init__(self, access_id, access_key, project_name, endpoint): + from odps import ODPS + self._odps = ODPS(access_id, access_key, project_name, endpoint) + + def _get_table(self, table_name): + """ + Get MaxCompute table object. + """ + return self._odps.get_table(table_name) + + def _read_data(self, table_name: str, pt_condition: str) -> pd.DataFrame: + """ + Read data from MaxCompute table. + :param table_name: table name + :param pt_condition: partition condition, + Example: pt_condition = 'dt=20230331' + :return: pandas dataframe with all data + """ + t = self._get_table(table_name) + + with t.open_reader(partition=pt_condition, limit=False) as reader: + pd_df = reader.to_pandas() + + return pd_df + + def fetch_data_to_csv(self, table_name: str, pt_condition: str, + output_path: str) -> None: + """ + Fetch data from MaxCompute table to local file. + :param table_name: table name + :param pt_condition: partition condition, + Example: pt_condition = 'dt=20230331' + :param output_path: output path + :return: None + """ + pd_df = self._read_data(table_name, pt_condition) + pd_df.to_csv(output_path, index=False) + print(f'Fetch data to {output_path} successfully.') + + @staticmethod + def _check_batch_args(reader, batch_size, limit): + if not limit: + limit = reader.count + if batch_size <= 0: + raise ValueError( + f'batch_size must be positive, but got {batch_size}') + if batch_size > limit: + batch_size = limit + return batch_size, limit + + @staticmethod + def gen_reader_batch(reader, batch_size_in: int, limit_in: int, + drop_last_in: bool, partitions: list, columns: list): + """ + Generate batch data from MaxCompute table. + + Args: + reader: MaxCompute table reader + batch_size_in: batch size + limit_in: limit of data, None means fetch all data + drop_last_in: whether drop last incomplete batch data + partitions: table partitions + columns: table columns + + Returns: + batch data generator + """ + + batch_size_in, limit_in = MaxComputeUtil._check_batch_args( + reader, batch_size_in, limit_in) + + batch_num = math.floor(limit_in / batch_size_in) + for i in range(batch_num + 1): + if i == batch_num and not drop_last_in and limit_in % batch_size_in > 0: + batch_records = reader[i * batch_size_in:( + i * batch_size_in + (limit_in % batch_size_in))] + else: + batch_records = reader[i * batch_size_in:(i + 1) + * batch_size_in] + batch_data_list = [] + for record in batch_records: + tmp_vals = [val for _, val in list(record)] + tmp_vals = tmp_vals[:(len(tmp_vals) - len(partitions))] + batch_data_list.append(tmp_vals) + yield pd.DataFrame(batch_data_list, columns=columns) + + @staticmethod + def gen_reader_item(reader, index: int, batch_size_in: int, limit_in: int, + drop_last_in: bool, partitions: list, columns: list): + """ + Get single batch data from MaxCompute table by indexing. + + Args: + reader: MaxCompute table reader + index: index of batch data + batch_size_in: batch size + limit_in: limit of data, None means fetch all data + drop_last_in: whether drop last incomplete batch data + partitions: table partitions + columns: table columns + + Returns: + single batch data (dataframe) + """ + batch_size_in, limit_in = MaxComputeUtil._check_batch_args( + reader, batch_size_in, limit_in) + + if drop_last_in: + batch_num = math.floor(limit_in / batch_size_in) + else: + batch_num = math.ceil(limit_in / batch_size_in) + + if index < 0: + raise ValueError(f'index must be non-negative, but got {index}') + if index >= batch_num: + raise ValueError( + f'index must be less than batch_num, but got index={index}, batch_num={batch_num}' + ) + + start = index * batch_size_in + end = (index + 1) * batch_size_in + if end > limit_in: + end = limit_in + batch_item = reader[start:end] + + batch_data_list = [] + for record in batch_item: + tmp_vals = [val for _, val in list(record)] + tmp_vals = tmp_vals[:(len(tmp_vals) - len(partitions))] + batch_data_list.append(tmp_vals) + + return pd.DataFrame(batch_data_list, columns=columns) + + def get_table_reader_ins(self, table_name: str, pt_condition: str = None): + + table_ins = self._get_table(table_name) + with table_ins.open_reader(partition=pt_condition) as reader: + return table_ins, reader diff --git a/modelscope/utils/constant.py b/modelscope/utils/constant.py index 5a23902f..929369ec 100644 --- a/modelscope/utils/constant.py +++ b/modelscope/utils/constant.py @@ -561,3 +561,17 @@ class VirgoDatasetConfig: col_analysis_result = 'analysis_result' col_external_info = 'external_info' col_cache_file = 'cache_file' + + +DEFAULT_MAXCOMPUTE_ENDPOINT = 'http://service-corp.odps.aliyun-inc.com/api' + + +class MaxComputeEnvs: + + ACCESS_ID = 'ODPS_ACCESS_ID' + + ACCESS_SECRET_KEY = 'ODPS_ACCESS_SECRET_KEY' + + PROJECT_NAME = 'ODPS_PROJECT_NAME' + + ENDPOINT = 'ODPS_ENDPOINT' diff --git a/tests/msdatasets/test_ms_dataset.py b/tests/msdatasets/test_ms_dataset.py index 52b015ec..ddb84b45 100644 --- a/tests/msdatasets/test_ms_dataset.py +++ b/tests/msdatasets/test_ms_dataset.py @@ -195,7 +195,7 @@ class MsDatasetTest(unittest.TestCase): ) print(next(iter(tf_dataset))) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_streaming_load_uni_fold(self): """Test case for loading large scale datasets.""" dataset = MsDataset.load( @@ -258,7 +258,7 @@ class MsDatasetTest(unittest.TestCase): def test_to_custom_dataset_movie_scene_toydata(self): from modelscope.msdatasets.dataset_cls.custom_datasets.movie_scene_segmentation import \ MovieSceneSegmentationDataset - from modelscope.msdatasets.dataset_cls.dataset import ExternalDataset + from modelscope.msdatasets.dataset_cls import ExternalDataset model_id = 'damo/cv_resnet50-bert_video-scene-segmentation_movienet' cache_path = snapshot_download(model_id) diff --git a/tests/msdatasets/test_virgo_dataset.py b/tests/msdatasets/test_virgo_dataset.py index 219b7052..96f7f25b 100644 --- a/tests/msdatasets/test_virgo_dataset.py +++ b/tests/msdatasets/test_virgo_dataset.py @@ -5,7 +5,7 @@ import unittest from modelscope.hub.api import HubApi from modelscope.msdatasets import MsDataset -from modelscope.msdatasets.dataset_cls import VirgoDataset +from modelscope.msdatasets.dataset_cls.dataset import VirgoDataset from modelscope.utils.constant import DownloadMode, Hubs, VirgoDatasetConfig from modelscope.utils.logger import get_logger @@ -71,6 +71,26 @@ class TestVirgoDataset(unittest.TestCase): cache_file_path = ds_one[VirgoDatasetConfig.col_cache_file] self.assertTrue(os.path.exists(cache_file_path)) + @unittest.skip('to be used for local test only') + def test_download_virgo_dataset_odps(self): + # Note: the samplingType must be 1, which means to get the dataset from MaxCompute(ODPS). + import pandas as pd + + ds = MsDataset.load( + dataset_name=VIRGO_DATASET_ID, + hub=Hubs.virgo, + odps_batch_size=100, + odps_limit=2000, + odps_drop_last=True) + + ds_one = next(iter(ds)) + logger.info(ds_one) + + self.assertTrue(ds_one) + self.assertIsInstance(ds, VirgoDataset) + self.assertTrue(ds_one, pd.DataFrame) + logger.info(f'The shape of sample: {ds_one.shape}') + if __name__ == '__main__': unittest.main() From 112466e06980d4972fab326d8766e65c1682d4ed Mon Sep 17 00:00:00 2001 From: "xingjun.wang" Date: Fri, 12 May 2023 17:37:58 +0800 Subject: [PATCH 29/76] update class ExtractiveSummarizationPipeline(Pipeline): --- .../pipelines/nlp/document_segmentation_pipeline.py | 10 +++------- .../pipelines/nlp/extractive_summarization_pipeline.py | 7 ++----- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/modelscope/pipelines/nlp/document_segmentation_pipeline.py b/modelscope/pipelines/nlp/document_segmentation_pipeline.py index 6e195ed0..d528eee0 100644 --- a/modelscope/pipelines/nlp/document_segmentation_pipeline.py +++ b/modelscope/pipelines/nlp/document_segmentation_pipeline.py @@ -6,11 +6,9 @@ from typing import Any, Dict, List, Union import numpy as np import torch from datasets import Dataset -from transformers.models.bert.modeling_bert import BertConfig from modelscope.metainfo import Pipelines from modelscope.models import Model -from modelscope.models.nlp.ponet.configuration import PoNetConfig from modelscope.outputs import OutputKeys from modelscope.pipelines.base import Pipeline, Tensor from modelscope.pipelines.builder import PIPELINES @@ -51,11 +49,9 @@ class DocumentSegmentationPipeline(Pipeline): auto_collate=auto_collate, **kwargs) - kwargs = kwargs - if 'compile' in kwargs.keys(): - kwargs.pop('compile') - if 'compile_options' in kwargs.keys(): - kwargs.pop('compile_options') + kwargs.pop('compile', None) + kwargs.pop('compile_options', None) + self.model_dir = self.model.model_dir self.model_cfg = self.model.model_cfg if preprocessor is None: diff --git a/modelscope/pipelines/nlp/extractive_summarization_pipeline.py b/modelscope/pipelines/nlp/extractive_summarization_pipeline.py index c01f28fc..a4e67607 100644 --- a/modelscope/pipelines/nlp/extractive_summarization_pipeline.py +++ b/modelscope/pipelines/nlp/extractive_summarization_pipeline.py @@ -44,11 +44,8 @@ class ExtractiveSummarizationPipeline(Pipeline): auto_collate=auto_collate, **kwargs) - kwargs = kwargs - if 'compile' in kwargs.keys(): - kwargs.pop('compile') - if 'compile_options' in kwargs.keys(): - kwargs.pop('compile_options') + kwargs.pop('compile', None) + kwargs.pop('compile_options', None) self.model_dir = self.model.model_dir self.model_cfg = self.model.model_cfg From 1cd67ef0607eb0a21191d59c5a8b54291fc02c9a Mon Sep 17 00:00:00 2001 From: "hejunjie.hjj" Date: Fri, 12 May 2023 17:51:00 +0800 Subject: [PATCH 30/76] [to #42322933] add cvpr23 fastinst model --- modelscope/metainfo.py | 3 +- .../image_instance_segmentation/__init__.py | 2 + .../backbones/__init__.py | 2 + .../backbones/resnet.py | 114 ++++++ .../fastinst/__init__.py | 1 + .../fastinst/fastinst_decoder.py | 351 ++++++++++++++++++ .../fastinst/fastinst_encoder.py | 180 +++++++++ .../fastinst_model.py | 221 +++++++++++ .../cv/fast_instance_segmentation_pipeline.py | 116 ++++++ .../test_fast_instance_segmentation.py | 39 ++ 10 files changed, 1028 insertions(+), 1 deletion(-) create mode 100644 modelscope/models/cv/image_instance_segmentation/backbones/resnet.py create mode 100644 modelscope/models/cv/image_instance_segmentation/fastinst/__init__.py create mode 100644 modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_decoder.py create mode 100644 modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_encoder.py create mode 100644 modelscope/models/cv/image_instance_segmentation/fastinst_model.py create mode 100644 modelscope/pipelines/cv/fast_instance_segmentation_pipeline.py create mode 100644 tests/pipelines/test_fast_instance_segmentation.py diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index f07192ea..22ad8b1a 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -116,6 +116,7 @@ class Models(object): bad_image_detecting = 'bad-image-detecting' controllable_image_generation = 'controllable-image-generation' longshortnet = 'longshortnet' + fastinst = 'fastinst' pedestrian_attribute_recognition = 'pedestrian-attribute-recognition' # nlp models @@ -396,7 +397,7 @@ class Pipelines(object): nerf_recon_acc = 'nerf-recon-acc' bad_image_detecting = 'bad-image-detecting' controllable_image_generation = 'controllable-image-generation' - + fast_instance_segmentation = 'fast-instance-segmentation' image_quality_assessment_mos = 'image-quality-assessment-mos' image_quality_assessment_man = 'image-quality-assessment-man' image_quality_assessment_degradation = 'image-quality-assessment-degradation' diff --git a/modelscope/models/cv/image_instance_segmentation/__init__.py b/modelscope/models/cv/image_instance_segmentation/__init__.py index 60e688eb..8041a7e7 100644 --- a/modelscope/models/cv/image_instance_segmentation/__init__.py +++ b/modelscope/models/cv/image_instance_segmentation/__init__.py @@ -8,10 +8,12 @@ if TYPE_CHECKING: from .maskdino_swin import MaskDINOSwin from .model import CascadeMaskRCNNSwinModel from .maskdino_model import MaskDINOSwinModel + from .fastinst_model import FastInst from .postprocess_utils import get_img_ins_seg_result, get_maskdino_ins_seg_result else: _import_structure = { 'cascade_mask_rcnn_swin': ['CascadeMaskRCNNSwin'], + 'fastinst_model': ['FastInst'], 'maskdino_swin': ['MaskDINOSwin'], 'model': ['CascadeMaskRCNNSwinModel'], 'maskdino_model': ['MaskDINOSwinModel'], diff --git a/modelscope/models/cv/image_instance_segmentation/backbones/__init__.py b/modelscope/models/cv/image_instance_segmentation/backbones/__init__.py index bbeac51e..1e7325f3 100644 --- a/modelscope/models/cv/image_instance_segmentation/backbones/__init__.py +++ b/modelscope/models/cv/image_instance_segmentation/backbones/__init__.py @@ -6,10 +6,12 @@ from modelscope.utils.import_utils import LazyImportModule if TYPE_CHECKING: from .swin_transformer import SwinTransformer from .swin_transformer import D2SwinTransformer + from .resnet import build_resnet_backbone else: _import_structure = { 'swin_transformer': ['SwinTransformer', 'D2SwinTransformer'], + 'resnet': ['build_resnet_backbone'] } import sys diff --git a/modelscope/models/cv/image_instance_segmentation/backbones/resnet.py b/modelscope/models/cv/image_instance_segmentation/backbones/resnet.py new file mode 100644 index 00000000..4e2a5ec1 --- /dev/null +++ b/modelscope/models/cv/image_instance_segmentation/backbones/resnet.py @@ -0,0 +1,114 @@ +# Part of the implementation is borrowed and modified from Detectron2, publicly available at +# https://github.com/facebookresearch/detectron2/blob/main/projects/DeepLab/deeplab/resnet.py + +import torch.nn.functional as F +from torch import nn + +from modelscope.models.cv.image_human_parsing.backbone.deeplab_resnet import ( + BottleneckBlock, DeeplabResNet, get_norm) +from modelscope.models.cv.image_instance_segmentation.maskdino.utils import \ + Conv2d + + +class BasicStem(nn.Module): + """ + The standard ResNet stem (layers before the first residual block), + with a conv, relu and max_pool. + """ + + def __init__(self, in_channels=3, out_channels=64, norm='BN'): + """ + Args: + norm (str or callable): norm after the first conv layer. + See :func:`layers.get_norm` for supported format. + """ + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = 4 + self.conv1 = Conv2d( + in_channels, + out_channels, + kernel_size=7, + stride=2, + padding=3, + bias=False, + norm=get_norm(norm, out_channels), + ) + + def forward(self, x): + x = self.conv1(x) + x = F.relu_(x) + x = F.max_pool2d(x, kernel_size=3, stride=2, padding=1) + return x + + +def build_resnet_backbone(out_features, depth, num_groups, width_per_group, + norm, stem_out_channels, res2_out_channels, + stride_in_1x1, res4_dilation, res5_dilation, + res5_multi_grid, input_shape): + stem = BasicStem( + in_channels=input_shape['channels'], + out_channels=stem_out_channels, + norm=norm) + bottleneck_channels = num_groups * width_per_group + in_channels = stem_out_channels + out_channels = res2_out_channels + + assert res4_dilation in { + 1, 2 + }, 'res4_dilation cannot be {}.'.format(res4_dilation) + assert res5_dilation in { + 1, 2, 4 + }, 'res5_dilation cannot be {}.'.format(res5_dilation) + if res4_dilation == 2: + # Always dilate res5 if res4 is dilated. + assert res5_dilation == 4 + + num_blocks_per_stage = { + 50: [3, 4, 6, 3], + 101: [3, 4, 23, 3], + 152: [3, 8, 36, 3] + }[depth] + + stages = [] + out_stage_idx = [{ + 'res2': 2, + 'res3': 3, + 'res4': 4, + 'res5': 5 + }[f] for f in out_features] + max_stage_idx = max(out_stage_idx) + for idx, stage_idx in enumerate(range(2, max_stage_idx + 1)): + if stage_idx == 4: + dilation = res4_dilation + elif stage_idx == 5: + dilation = res5_dilation + else: + dilation = 1 + first_stride = 1 if idx == 0 or dilation > 1 else 2 + stride_per_block = [first_stride] + stride_per_block += [1] * (num_blocks_per_stage[idx] - 1) + stage_kargs = { + 'num_blocks': num_blocks_per_stage[idx], + 'stride_per_block': stride_per_block, + 'in_channels': in_channels, + 'out_channels': out_channels, + 'norm': norm, + 'bottleneck_channels': bottleneck_channels, + 'stride_in_1x1': stride_in_1x1, + 'dilation': dilation, + 'num_groups': num_groups, + 'block_class': BottleneckBlock + } + if stage_idx == 5: + stage_kargs.pop('dilation') + stage_kargs['dilation_per_block'] = [ + dilation * mg for mg in res5_multi_grid + ] + blocks = DeeplabResNet.make_stage(**stage_kargs) + in_channels = out_channels + out_channels *= 2 + bottleneck_channels *= 2 + stages.append(blocks) + return DeeplabResNet(stem, stages, out_features=out_features) diff --git a/modelscope/models/cv/image_instance_segmentation/fastinst/__init__.py b/modelscope/models/cv/image_instance_segmentation/fastinst/__init__.py new file mode 100644 index 00000000..b937315b --- /dev/null +++ b/modelscope/models/cv/image_instance_segmentation/fastinst/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_decoder.py b/modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_decoder.py new file mode 100644 index 00000000..aa4300f6 --- /dev/null +++ b/modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_decoder.py @@ -0,0 +1,351 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import math + +import torch +from torch import nn +from torch.nn import functional as F + +from modelscope.models.cv.image_colorization.ddcolor.utils.transformer_utils import ( + MLP, CrossAttentionLayer, FFNLayer, SelfAttentionLayer) + + +class QueryProposal(nn.Module): + + def __init__(self, num_features, num_queries, num_classes): + super().__init__() + self.topk = num_queries + self.num_classes = num_classes + + self.conv_proposal_cls_logits = nn.Sequential( + nn.Conv2d( + num_features, num_features, kernel_size=3, stride=1, + padding=1), + nn.ReLU(inplace=True), + nn.Conv2d( + num_features, + num_classes + 1, + kernel_size=1, + stride=1, + padding=0), + ) + + @torch.no_grad() + def compute_coordinates(self, x): + h, w = x.size(2), x.size(3) + y_loc = torch.linspace(0, 1, h, device=x.device) + x_loc = torch.linspace(0, 1, w, device=x.device) + y_loc, x_loc = torch.meshgrid(y_loc, x_loc) + locations = torch.stack([x_loc, y_loc], 0).unsqueeze(0) + return locations + + def seek_local_maximum(self, x, epsilon=1e-6): + """ + inputs: + x: torch.tensor, shape [b, c, h, w] + return: + torch.tensor, shape [b, c, h, w] + """ + x_pad = F.pad(x, (1, 1, 1, 1), 'constant', 0) + # top, bottom, left, right, top-left, top-right, bottom-left, bottom-right + maximum = (x >= x_pad[:, :, :-2, 1:-1]) & \ + (x >= x_pad[:, :, 2:, 1:-1]) & \ + (x >= x_pad[:, :, 1:-1, :-2]) & \ + (x >= x_pad[:, :, 1:-1, 2:]) & \ + (x >= x_pad[:, :, :-2, :-2]) & \ + (x >= x_pad[:, :, :-2, 2:]) & \ + (x >= x_pad[:, :, 2:, :-2]) & \ + (x >= x_pad[:, :, 2:, 2:]) & \ + (x >= epsilon) + return maximum.to(x) + + def forward(self, x, pos_embeddings): + + proposal_cls_logits = self.conv_proposal_cls_logits(x) # b, c, h, w + proposal_cls_probs = proposal_cls_logits.softmax(dim=1) # b, c, h, w + proposal_cls_one_hot = F.one_hot( + proposal_cls_probs[:, :-1, :, :].max(1)[1], + num_classes=self.num_classes + 1).permute(0, 3, 1, 2) # b, c, h, w + proposal_cls_probs = proposal_cls_probs.mul(proposal_cls_one_hot) + proposal_local_maximum_map = self.seek_local_maximum( + proposal_cls_probs) # b, c, h, w + proposal_cls_probs = proposal_cls_probs + proposal_local_maximum_map # b, c, h, w + + # top-k indices + topk_indices = torch.topk( + proposal_cls_probs[:, :-1, :, :].flatten(2).max(1)[0], + self.topk, + dim=1)[1] # b, q + topk_indices = topk_indices.unsqueeze(1) # b, 1, q + + # topk queries + topk_proposals = torch.gather( + x.flatten(2), dim=2, index=topk_indices.repeat(1, x.shape[1], + 1)) # b, c, q + pos_embeddings = pos_embeddings.repeat(x.shape[0], 1, 1, 1).flatten(2) + topk_pos_embeddings = torch.gather( + pos_embeddings, + dim=2, + index=topk_indices.repeat(1, pos_embeddings.shape[1], + 1)) # b, c, q + if self.training: + locations = self.compute_coordinates(x).repeat(x.shape[0], 1, 1, 1) + topk_locations = torch.gather( + locations.flatten(2), + dim=2, + index=topk_indices.repeat(1, locations.shape[1], 1)) + topk_locations = topk_locations.transpose(-1, -2) # b, q, 2 + else: + topk_locations = None + return topk_proposals, topk_pos_embeddings, topk_locations, proposal_cls_logits + + +class FastInstDecoder(nn.Module): + + def __init__(self, in_channels, *, num_classes: int, hidden_dim: int, + num_queries: int, num_aux_queries: int, nheads: int, + dim_feedforward: int, dec_layers: int, pre_norm: bool, + mask_dim: int): + """ + Args: + in_channels: channels of the input features + num_classes: number of classes + hidden_dim: Transformer feature dimension + num_queries: number of queries + num_aux_queries: number of auxiliary queries + nheads: number of heads + dim_feedforward: feature dimension in feedforward network + dec_layers: number of Transformer decoder layers + pre_norm: whether to use pre-LayerNorm or not + mask_dim: mask feature dimension + """ + super().__init__() + self.num_heads = nheads + self.num_layers = dec_layers + self.num_queries = num_queries + self.num_aux_queries = num_aux_queries + self.num_classes = num_classes + + meta_pos_size = int(round(math.sqrt(self.num_queries))) + self.meta_pos_embed = nn.Parameter( + torch.empty(1, hidden_dim, meta_pos_size, meta_pos_size)) + if num_aux_queries > 0: + self.empty_query_features = nn.Embedding(num_aux_queries, + hidden_dim) + self.empty_query_pos_embed = nn.Embedding(num_aux_queries, + hidden_dim) + + self.query_proposal = QueryProposal(hidden_dim, num_queries, + num_classes) + + self.transformer_query_cross_attention_layers = nn.ModuleList() + self.transformer_query_self_attention_layers = nn.ModuleList() + self.transformer_query_ffn_layers = nn.ModuleList() + self.transformer_mask_cross_attention_layers = nn.ModuleList() + self.transformer_mask_ffn_layers = nn.ModuleList() + for idx in range(self.num_layers): + self.transformer_query_cross_attention_layers.append( + CrossAttentionLayer( + d_model=hidden_dim, + nhead=nheads, + dropout=0.0, + normalize_before=pre_norm)) + self.transformer_query_self_attention_layers.append( + SelfAttentionLayer( + d_model=hidden_dim, + nhead=nheads, + dropout=0.0, + normalize_before=pre_norm)) + self.transformer_query_ffn_layers.append( + FFNLayer( + d_model=hidden_dim, + dim_feedforward=dim_feedforward, + dropout=0.0, + normalize_before=pre_norm)) + self.transformer_mask_cross_attention_layers.append( + CrossAttentionLayer( + d_model=hidden_dim, + nhead=nheads, + dropout=0.0, + normalize_before=pre_norm)) + self.transformer_mask_ffn_layers.append( + FFNLayer( + d_model=hidden_dim, + dim_feedforward=dim_feedforward, + dropout=0.0, + normalize_before=pre_norm)) + + self.decoder_query_norm_layers = nn.ModuleList() + self.class_embed_layers = nn.ModuleList() + self.mask_embed_layers = nn.ModuleList() + self.mask_features_layers = nn.ModuleList() + for idx in range(self.num_layers + 1): + self.decoder_query_norm_layers.append(nn.LayerNorm(hidden_dim)) + self.class_embed_layers.append( + MLP(hidden_dim, hidden_dim, num_classes + 1, 3)) + self.mask_embed_layers.append( + MLP(hidden_dim, hidden_dim, mask_dim, 3)) + self.mask_features_layers.append(nn.Linear(hidden_dim, mask_dim)) + + def forward(self, x, mask_features, targets=None): + bs = x[0].shape[0] + proposal_size = x[1].shape[-2:] + pixel_feature_size = x[2].shape[-2:] + + pixel_pos_embeds = F.interpolate( + self.meta_pos_embed, + size=pixel_feature_size, + mode='bilinear', + align_corners=False) + proposal_pos_embeds = F.interpolate( + self.meta_pos_embed, + size=proposal_size, + mode='bilinear', + align_corners=False) + + pixel_features = x[2].flatten(2).permute(2, 0, 1) + pixel_pos_embeds = pixel_pos_embeds.flatten(2).permute(2, 0, 1) + + query_features, query_pos_embeds, query_locations, proposal_cls_logits = self.query_proposal( + x[1], proposal_pos_embeds) + query_features = query_features.permute(2, 0, 1) + query_pos_embeds = query_pos_embeds.permute(2, 0, 1) + if self.num_aux_queries > 0: + aux_query_features = self.empty_query_features.weight.unsqueeze( + 1).repeat(1, bs, 1) + aux_query_pos_embed = self.empty_query_pos_embed.weight.unsqueeze( + 1).repeat(1, bs, 1) + query_features = torch.cat([query_features, aux_query_features], + dim=0) + query_pos_embeds = torch.cat( + [query_pos_embeds, aux_query_pos_embed], dim=0) + + outputs_class, outputs_mask, attn_mask, _, _ = self.forward_prediction_heads( + query_features, + pixel_features, + pixel_feature_size, + -1, + return_attn_mask=True) + predictions_class = [outputs_class] + predictions_mask = [outputs_mask] + predictions_matching_index = [None] + query_feature_memory = [query_features] + pixel_feature_memory = [pixel_features] + + for i in range(self.num_layers): + query_features, pixel_features = self.forward_one_layer( + query_features, pixel_features, query_pos_embeds, + pixel_pos_embeds, attn_mask, i) + if i < self.num_layers - 1: + outputs_class, outputs_mask, attn_mask, _, _ = self.forward_prediction_heads( + query_features, + pixel_features, + pixel_feature_size, + i, + return_attn_mask=True, + ) + else: + outputs_class, outputs_mask, _, matching_indices, gt_attn_mask = self.forward_prediction_heads( + query_features, + pixel_features, + pixel_feature_size, + i, + ) + predictions_class.append(outputs_class) + predictions_mask.append(outputs_mask) + predictions_matching_index.append(None) + query_feature_memory.append(query_features) + pixel_feature_memory.append(pixel_features) + + out = { + 'proposal_cls_logits': + proposal_cls_logits, + 'query_locations': + query_locations, + 'pred_logits': + predictions_class[-1], + 'pred_masks': + predictions_mask[-1], + 'pred_indices': + predictions_matching_index[-1], + 'aux_outputs': + self._set_aux_loss(predictions_class, predictions_mask, + predictions_matching_index, query_locations) + } + return out + + def forward_one_layer(self, query_features, pixel_features, + query_pos_embeds, pixel_pos_embeds, attn_mask, i): + pixel_features = self.transformer_mask_cross_attention_layers[i]( + pixel_features, + query_features, + query_pos=pixel_pos_embeds, + pos=query_pos_embeds) + pixel_features = self.transformer_mask_ffn_layers[i](pixel_features) + + query_features = self.transformer_query_cross_attention_layers[i]( + query_features, + pixel_features, + memory_mask=attn_mask, + query_pos=query_pos_embeds, + pos=pixel_pos_embeds) + query_features = self.transformer_query_self_attention_layers[i]( + query_features, query_pos=query_pos_embeds) + query_features = self.transformer_query_ffn_layers[i](query_features) + return query_features, pixel_features + + def forward_prediction_heads(self, + query_features, + pixel_features, + pixel_feature_size, + idx_layer, + return_attn_mask=False, + return_gt_attn_mask=False, + targets=None, + query_locations=None): + decoder_query_features = self.decoder_query_norm_layers[idx_layer + 1]( + query_features[:self.num_queries]) + decoder_query_features = decoder_query_features.transpose(0, 1) + if idx_layer + 1 == self.num_layers: + outputs_class = self.class_embed_layers[idx_layer + 1]( + decoder_query_features) + else: + outputs_class = None + outputs_mask_embed = self.mask_embed_layers[idx_layer + 1]( + decoder_query_features) + outputs_mask_features = self.mask_features_layers[idx_layer + 1]( + pixel_features.transpose(0, 1)) + + outputs_mask = torch.einsum('bqc,blc->bql', outputs_mask_embed, + outputs_mask_features) + outputs_mask = outputs_mask.reshape(-1, self.num_queries, + *pixel_feature_size) + + if return_attn_mask: + # outputs_mask.shape: b, q, h, w + attn_mask = F.pad(outputs_mask, + (0, 0, 0, 0, 0, self.num_aux_queries), + 'constant', 1) + attn_mask = (attn_mask < 0.).flatten(2) # b, q, hw + invalid_query = attn_mask.all(-1, keepdim=True) # b, q, 1 + attn_mask = (~invalid_query) & attn_mask # b, q, hw + attn_mask = attn_mask.unsqueeze(1).repeat(1, self.num_heads, 1, + 1).flatten(0, 1) + attn_mask = attn_mask.detach() + else: + attn_mask = None + + matching_indices = None + gt_attn_mask = None + + return outputs_class, outputs_mask, attn_mask, matching_indices, gt_attn_mask + + @torch.jit.unused + def _set_aux_loss(self, outputs_class, outputs_seg_masks, output_indices, + output_query_locations): + return [{ + 'query_locations': output_query_locations, + 'pred_logits': a, + 'pred_masks': b, + 'pred_matching_indices': c + } for a, b, c in zip(outputs_class[:-1], outputs_seg_masks[:-1], + output_indices[:-1])] diff --git a/modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_encoder.py b/modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_encoder.py new file mode 100644 index 00000000..46b3f74d --- /dev/null +++ b/modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_encoder.py @@ -0,0 +1,180 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import logging +from typing import Callable, Optional, Union + +import torch +from torch import nn +from torch.nn import functional as F + +from modelscope.models.cv.image_instance_segmentation.maskdino.utils import \ + Conv2d + + +# This is a modified FPN decoder. +class BaseFPN(nn.Module): + + def __init__( + self, + input_shape, + *, + convs_dim: int, + mask_dim: int, + norm: Optional[Union[str, Callable]] = None, + ): + """ + Args: + input_shape: shapes (channels and stride) of the input features + convs_dim: number of output channels for the intermediate conv layers. + mask_dim: number of output channels for the final conv layer. + norm (str or callable): normalization for all conv layers + """ + super().__init__() + + input_shape = sorted(input_shape.items(), key=lambda x: x[1]['stride']) + self.in_features = [k for k, v in input_shape + ] # starting from "res3" to "res5" + feature_channels = [v['channels'] for k, v in input_shape] + + lateral_convs = [] + output_convs = [] + + use_bias = norm == '' + for idx, in_channels in enumerate(feature_channels): + lateral_norm = nn.GroupNorm(32, convs_dim) + output_norm = nn.GroupNorm(32, convs_dim) + + lateral_conv = Conv2d( + in_channels, + convs_dim, + kernel_size=1, + bias=use_bias, + norm=lateral_norm) + output_conv = Conv2d( + convs_dim, + convs_dim, + kernel_size=3, + stride=1, + padding=1, + bias=use_bias, + norm=output_norm, + activation=F.relu, + ) + self.add_module('adapter_{}'.format(idx + 1), lateral_conv) + self.add_module('layer_{}'.format(idx + 1), output_conv) + + lateral_convs.append(lateral_conv) + output_convs.append(output_conv) + # Place convs into top-down order (from low to high resolution) + # to make the top-down computation in forward clearer. + self.lateral_convs = lateral_convs[::-1] + self.output_convs = output_convs[::-1] + + self.convs_dim = convs_dim + self.num_feature_levels = 3 # always use 3 scales + + def forward_features(self, features): + multi_scale_features = [] + num_cur_levels = 0 + # Reverse feature maps into top-down order (from low to high resolution) + for idx, f in enumerate(self.in_features[::-1]): + x = features[f] + lateral_conv = self.lateral_convs[idx] + output_conv = self.output_convs[idx] + if idx == 0: + y = lateral_conv(x) + else: + cur_fpn = lateral_conv(x) + y = cur_fpn + F.interpolate( + y, + size=cur_fpn.shape[-2:], + mode='bilinear', + align_corners=False) + y = output_conv(y) + + if num_cur_levels < self.num_feature_levels: + multi_scale_features.append(y) + num_cur_levels += 1 + return None, multi_scale_features + + def forward(self, features, targets=None): + logger = logging.getLogger(__name__) + logger.warning( + 'Calling forward() may cause unpredicted behavior of PixelDecoder module.' + ) + return self.forward_features(features) + + +class PyramidPoolingModule(nn.Module): + + def __init__(self, in_channels, channels=512, sizes=(1, 2, 3, 6)): + super().__init__() + self.stages = [] + self.stages = nn.ModuleList( + [self._make_stage(in_channels, channels, size) for size in sizes]) + self.bottleneck = Conv2d(in_channels + len(sizes) * channels, + in_channels, 1) + + def _make_stage(self, features, out_features, size): + prior = nn.AdaptiveAvgPool2d(output_size=(size, size)) + conv = Conv2d(features, out_features, 1) + return nn.Sequential(prior, conv) + + def forward(self, feats): + h, w = feats.size(2), feats.size(3) + priors = [ + F.interpolate( + input=F.relu_(stage(feats)), + size=(h, w), + mode='bilinear', + align_corners=False) for stage in self.stages + ] + [feats] + out = F.relu_(self.bottleneck(torch.cat(priors, 1))) + return out + + +class PyramidPoolingModuleFPN(BaseFPN): + + def __init__( + self, + input_shape, + *, + convs_dim: int, + mask_dim: int, + norm: Optional[Union[str, Callable]] = None, + ): + """ + NOTE: this interface is experimental. + Args: + input_shape: shapes (channels and stride) of the input features + convs_dim: number of output channels for the intermediate conv layers. + mask_dim: number of output channels for the final conv layer. + norm (str or callable): normalization for all conv layers + """ + super().__init__( + input_shape, convs_dim=convs_dim, mask_dim=mask_dim, norm=norm) + self.ppm = PyramidPoolingModule(convs_dim, convs_dim // 4) + + def forward_features(self, features): + multi_scale_features = [] + num_cur_levels = 0 + # Reverse feature maps into top-down order (from low to high resolution) + for idx, f in enumerate(self.in_features[::-1]): + x = features[f] + lateral_conv = self.lateral_convs[idx] + output_conv = self.output_convs[idx] + if idx == 0: + y = self.ppm(lateral_conv(x)) + else: + cur_fpn = lateral_conv(x) + y = cur_fpn + F.interpolate( + y, + size=cur_fpn.shape[-2:], + mode='bilinear', + align_corners=False) + y = output_conv(y) + + if num_cur_levels < self.num_feature_levels: + multi_scale_features.append(y) + num_cur_levels += 1 + + return None, multi_scale_features diff --git a/modelscope/models/cv/image_instance_segmentation/fastinst_model.py b/modelscope/models/cv/image_instance_segmentation/fastinst_model.py new file mode 100644 index 00000000..f9cfbc4f --- /dev/null +++ b/modelscope/models/cv/image_instance_segmentation/fastinst_model.py @@ -0,0 +1,221 @@ +# Part of implementation is borrowed and modified from Mask2Former, publicly available at +# https://github.com/facebookresearch/Mask2Former. +import os +from typing import Any, Dict, List + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.image_instance_segmentation.maskdino_swin import \ + ImageList +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .backbones import build_resnet_backbone +from .fastinst.fastinst_decoder import FastInstDecoder +from .fastinst.fastinst_encoder import PyramidPoolingModuleFPN + +logger = get_logger() + + +@MODELS.register_module(Tasks.image_segmentation, module_name=Models.fastinst) +class FastInst(TorchModel): + + def __init__(self, + model_dir, + backbone=None, + encoder=None, + decoder=None, + pretrained=None, + classes=None, + **kwargs): + """ + Deep Learning Technique for Human Parsing: A Survey and Outlook. See https://arxiv.org/abs/2301.00394 + Args: + backbone (dict): backbone config. + encoder (dict): encoder config. + decoder (dict): decoder config. + pretrained (bool): whether to use pretrained model + classes (list): class names + """ + super(FastInst, self).__init__(model_dir, **kwargs) + + self.backbone = build_resnet_backbone( + **backbone, input_shape={'channels': 3}) + in_features = encoder.pop('in_features') + input_shape = { + k: v + for k, v in self.backbone.output_shape().items() + if k in in_features + } + encoder = PyramidPoolingModuleFPN(input_shape=input_shape, **encoder) + decoder = FastInstDecoder(in_channels=encoder.convs_dim, **decoder) + self.sem_seg_head = FastInstHead( + pixel_decoder=encoder, transformer_predictor=decoder) + + self.num_classes = decoder.num_classes + self.num_queries = decoder.num_queries + self.size_divisibility = 32 + self.register_buffer( + 'pixel_mean', + torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1), False) + self.register_buffer( + 'pixel_std', + torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1), False) + self.classes = classes + self.test_topk_per_image = 100 + + if pretrained: + model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + logger.info(f'loading model from {model_path}') + weight = torch.load(model_path, map_location='cpu')['model'] + tgt_weight = self.state_dict() + for name in list(weight.keys()): + if name in tgt_weight: + load_size = weight[name].size() + tgt_size = tgt_weight[name].size() + mis_match = False + if len(load_size) != len(tgt_size): + mis_match = True + else: + for n1, n2 in zip(load_size, tgt_size): + if n1 != n2: + mis_match = True + break + if mis_match: + logger.info( + f'size mismatch for {name} ' + f'({load_size} -> {tgt_size}), skip loading.') + del weight[name] + else: + logger.info( + f'{name} doesn\'t exist in current model, skip loading.' + ) + + self.load_state_dict(weight, strict=False) + logger.info('load model done') + + def forward(self, batched_inputs: List[dict]) -> Dict[str, Any]: + images = [x['image'].to(self.device) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors(images, self.size_divisibility) + + features = self.backbone(images.tensor) + outputs = self.sem_seg_head(features) + + return dict( + outputs=outputs, batched_inputs=batched_inputs, images=images) + + def postprocess(self, input: Dict[str, Any]) -> Dict[str, Any]: + outputs = input['outputs'] + batched_inputs = input['batched_inputs'] + images = input['images'] + if self.training: + raise NotImplementedError + else: + mask_cls_results = outputs['pred_logits'] # (B, Q, C+1) + mask_pred_results = outputs['pred_masks'] # (B, Q, H, W) + # upsample masks + mask_pred_results = F.interpolate( + mask_pred_results, + size=(images.tensor.shape[-2], images.tensor.shape[-1]), + mode='bilinear', + align_corners=False, + ) + + del outputs + + processed_results = [] + for mask_cls_result, mask_pred_result, input_per_image, image_size in zip( + mask_cls_results, mask_pred_results, batched_inputs, + images.image_sizes): + height = input_per_image.get('height', image_size[0]) + width = input_per_image.get('width', image_size[1]) + processed_results.append({}) # for each image + + mask_pred_result = self.sem_seg_postprocess( + mask_pred_result, image_size, height, width) + mask_cls_result = mask_cls_result.to(mask_pred_result) + + instance_r = self.instance_inference(mask_cls_result, + mask_pred_result) + processed_results[-1]['instances'] = instance_r + + return dict(eval_result=processed_results) + + @property + def device(self): + return self.pixel_mean.device + + def sem_seg_postprocess(self, result, img_size, output_height, + output_width): + result = result[:, :img_size[0], :img_size[1]].expand(1, -1, -1, -1) + result = F.interpolate( + result, + size=(output_height, output_width), + mode='bilinear', + align_corners=False)[0] + return result + + def instance_inference(self, mask_cls, mask_pred): + # mask_pred is already processed to have the same shape as original input + image_size = mask_pred.shape[-2:] + + # [Q, K] + scores = F.softmax(mask_cls, dim=-1)[:, :-1] + labels = torch.arange( + self.num_classes, + device=self.device).unsqueeze(0).repeat(self.num_queries, + 1).flatten(0, 1) + scores_per_image, topk_indices = scores.flatten(0, 1).topk( + self.test_topk_per_image, sorted=False) + labels_per_image = labels[topk_indices] + + topk_indices = topk_indices // self.num_classes + mask_pred = mask_pred[topk_indices] + + result = {'image_size': image_size} + # mask (before sigmoid) + mask_pred_sigmoid = mask_pred.sigmoid() + result['pred_masks'] = (mask_pred_sigmoid > 0.5).float() + + # calculate average mask prob + mask_scores_per_image = (mask_pred_sigmoid.flatten(1) + * result['pred_masks'].flatten(1)).sum(1) / ( + result['pred_masks'].flatten(1).sum(1) + + 1e-6) + result['scores'] = scores_per_image * mask_scores_per_image + result['pred_classes'] = labels_per_image + return result + + +class FastInstHead(nn.Module): + + def __init__( + self, + *, + pixel_decoder: nn.Module, + # extra parameters + transformer_predictor: nn.Module): + """ + NOTE: this interface is experimental. + Args: + pixel_decoder: the pixel decoder module + transformer_predictor: the transformer decoder that makes prediction + """ + super().__init__() + self.pixel_decoder = pixel_decoder + self.predictor = transformer_predictor + + def forward(self, features, targets=None): + return self.layers(features, targets) + + def layers(self, features, targets=None): + mask_features, multi_scale_features = self.pixel_decoder.forward_features( + features) + predictions = self.predictor(multi_scale_features, mask_features, + targets) + return predictions diff --git a/modelscope/pipelines/cv/fast_instance_segmentation_pipeline.py b/modelscope/pipelines/cv/fast_instance_segmentation_pipeline.py new file mode 100644 index 00000000..6ee341de --- /dev/null +++ b/modelscope/pipelines/cv/fast_instance_segmentation_pipeline.py @@ -0,0 +1,116 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Any, Dict, Optional, Union + +import numpy as np +import torch +import torchvision.transforms as T + +from modelscope.metainfo import Pipelines +from modelscope.models.cv.image_instance_segmentation import FastInst +from modelscope.outputs import OutputKeys +from modelscope.pipelines.base import Input, Pipeline +from modelscope.pipelines.builder import PIPELINES +from modelscope.preprocessors import LoadImage +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@PIPELINES.register_module( + Tasks.image_segmentation, module_name=Pipelines.fast_instance_segmentation) +class FastInstanceSegmentationPipeline(Pipeline): + + def __init__(self, + model: Union[FastInst, str], + preprocessor: Optional = None, + **kwargs): + r"""The inference pipeline for fastinst models. + + The model outputs a dict with keys of `scores`, `labels`, and `masks`. + + Args: + model (`str` or `Model` or module instance): A model instance or a model local dir + or a model id in the model hub. + preprocessor (`Preprocessor`, `optional`): A Preprocessor instance. + kwargs (dict, `optional`): + Extra kwargs passed into the preprocessor's constructor. + + Examples: + >>> from modelscope.outputs import OutputKeys + >>> from modelscope.pipelines import pipeline + >>> pipeline_ins = pipeline('image-segmentation', + model='damo/cv_resnet50_fast-instance-segmentation_coco') + >>> input_img = 'https://modelscope.oss-cn-beijing.aliyuncs.com/test/images/image_instance_segmentation.jpg' + >>> print(pipeline_ins(input_img)[OutputKeys.LABELS]) + """ + super().__init__(model=model, preprocessor=preprocessor, **kwargs) + self.model.eval() + + def _get_preprocess_shape(self, oldh, oldw, short_edge_length, max_size): + h, w = oldh, oldw + size = short_edge_length * 1.0 + scale = size / min(h, w) + if h < w: + newh, neww = size, scale * w + else: + newh, neww = scale * h, size + if max(newh, neww) > max_size: + scale = max_size * 1.0 / max(newh, neww) + newh = newh * scale + neww = neww * scale + neww = int(neww + 0.5) + newh = int(newh + 0.5) + return (newh, neww) + + def preprocess(self, + input: Input, + min_size=640, + max_size=1333) -> Dict[str, Any]: + image = LoadImage.convert_to_img(input) + w, h = image.size[:2] + dataset_dict = {'width': w, 'height': h} + new_h, new_w = self._get_preprocess_shape(h, w, min_size, max_size) + test_transforms = T.Compose([ + T.Resize((new_h, new_w)), + T.ToTensor(), + ]) + image = test_transforms(image) + dataset_dict['image'] = image * 255. + result = {'batched_inputs': [dataset_dict]} + return result + + def forward(self, input: Dict[str, Any], + **forward_params) -> Dict[str, Any]: + with torch.no_grad(): + output = self.model(**input) + return output + + def postprocess(self, + inputs: Dict[str, Any], + score_thr=0.5) -> Dict[str, Any]: + predictions = inputs['eval_result'][0]['instances'] + scores = predictions['scores'].detach().cpu().numpy() + pred_masks = predictions['pred_masks'].detach().cpu().numpy() + pred_classes = predictions['pred_classes'].detach().cpu().numpy() + + thresholded_idxs = np.array(scores) >= score_thr + scores = scores[thresholded_idxs] + pred_classes = pred_classes[thresholded_idxs] + pred_masks = pred_masks[thresholded_idxs] + + results_dict = { + OutputKeys.MASKS: [], + OutputKeys.LABELS: [], + OutputKeys.SCORES: [] + } + for score, cls, mask in zip(scores, pred_classes, pred_masks): + score = np.float64(score) + label = self.model.classes[int(cls)] + mask = np.array(mask, dtype=np.float64) + + results_dict[OutputKeys.SCORES].append(score) + results_dict[OutputKeys.LABELS].append(label) + results_dict[OutputKeys.MASKS].append(mask) + + return results_dict diff --git a/tests/pipelines/test_fast_instance_segmentation.py b/tests/pipelines/test_fast_instance_segmentation.py new file mode 100644 index 00000000..aefd1092 --- /dev/null +++ b/tests/pipelines/test_fast_instance_segmentation.py @@ -0,0 +1,39 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import unittest + +from modelscope.models import Model +from modelscope.outputs import OutputKeys +from modelscope.pipelines import pipeline +from modelscope.utils.constant import Tasks +from modelscope.utils.demo_utils import DemoCompatibilityCheck +from modelscope.utils.test_utils import test_level + + +class FastInstanceSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): + + def setUp(self) -> None: + self.task = Tasks.image_segmentation + self.model_id = 'damo/cv_resnet50_fast-instance-segmentation_coco' + + image = 'data/test/images/image_instance_segmentation.jpg' + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_name(self): + pipeline_parsing = pipeline( + task=Tasks.image_segmentation, model=self.model_id) + print(pipeline_parsing(input=self.image)[OutputKeys.LABELS]) + + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') + def test_run_with_model_from_modelhub(self): + model = Model.from_pretrained(self.model_id) + pipeline_parsing = pipeline( + task=Tasks.image_segmentation, model=model, preprocessor=None) + print(pipeline_parsing(input=self.image)[OutputKeys.LABELS]) + + @unittest.skip('demo compatibility test is only enabled on a needed-basis') + def test_demo_compatibility(self): + self.compatibility_check() + + +if __name__ == '__main__': + unittest.main() From ca0cf2b5cfe47a1f73d60250915351dfaab33181 Mon Sep 17 00:00:00 2001 From: "tongmu.wh" Date: Fri, 12 May 2023 17:53:56 +0800 Subject: [PATCH 31/76] =?UTF-8?q?merge=20master=20,=E5=8A=A0=E5=85=A5speak?= =?UTF-8?q?er=20change=20locating=20pipeline=20Link:=20https://code.alibab?= =?UTF-8?q?a-inc.com/Ali-MaaS/MaaS-lib/codereview/12601179?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modelscope/metainfo.py | 2 + modelscope/models/audio/sv/DTDNN.py | 18 +- .../models/audio/sv/speaker_change_locator.py | 319 ++++++++++++++++++ .../audio/speaker_change_locating_pipeline.py | 105 ++++++ tests/pipelines/test_speaker_verification.py | 20 +- 5 files changed, 457 insertions(+), 7 deletions(-) create mode 100644 modelscope/models/audio/sv/speaker_change_locator.py create mode 100644 modelscope/pipelines/audio/speaker_change_locating_pipeline.py diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index 22ad8b1a..efce05b0 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -182,6 +182,7 @@ class Models(object): generic_sv = 'generic-sv' ecapa_tdnn_sv = 'ecapa-tdnn-sv' campplus_sv = 'cam++-sv' + scl_sd = 'scl-sd' rdino_tdnn_sv = 'rdino_ecapa-tdnn-sv' generic_lm = 'generic-lm' @@ -481,6 +482,7 @@ class Pipelines(object): vad_inference = 'vad-inference' speaker_verification = 'speaker-verification' speaker_verification_rdino = 'speaker-verification-rdino' + speaker_change_locating = 'speaker-change-locating' lm_inference = 'language-score-prediction' speech_timestamp_inference = 'speech-timestamp-inference' diff --git a/modelscope/models/audio/sv/DTDNN.py b/modelscope/models/audio/sv/DTDNN.py index d9e21ce8..d86d6799 100644 --- a/modelscope/models/audio/sv/DTDNN.py +++ b/modelscope/models/audio/sv/DTDNN.py @@ -76,11 +76,13 @@ class CAMPPlus(nn.Module): bn_size=4, init_channels=128, config_str='batchnorm-relu', - memory_efficient=True): + memory_efficient=True, + output_level='segment'): super(CAMPPlus, self).__init__() self.head = FCM(feat_dim=feat_dim) channels = self.head.out_channels + self.output_level = output_level self.xvector = nn.Sequential( OrderedDict([ @@ -118,10 +120,14 @@ class CAMPPlus(nn.Module): self.xvector.add_module('out_nonlinear', get_nonlinear(config_str, channels)) - self.xvector.add_module('stats', StatsPool()) - self.xvector.add_module( - 'dense', - DenseLayer(channels * 2, embedding_size, config_str='batchnorm_')) + if self.output_level == 'segment': + self.xvector.add_module('stats', StatsPool()) + self.xvector.add_module( + 'dense', + DenseLayer( + channels * 2, embedding_size, config_str='batchnorm_')) + else: + assert self.output_level == 'frame', '`output_level` should be set to \'segment\' or \'frame\'. ' for m in self.modules(): if isinstance(m, (nn.Conv1d, nn.Linear)): @@ -133,6 +139,8 @@ class CAMPPlus(nn.Module): x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T) x = self.head(x) x = self.xvector(x) + if self.output_level == 'frame': + x = x.transpose(1, 2) return x diff --git a/modelscope/models/audio/sv/speaker_change_locator.py b/modelscope/models/audio/sv/speaker_change_locator.py new file mode 100644 index 00000000..c22e4c1b --- /dev/null +++ b/modelscope/models/audio/sv/speaker_change_locator.py @@ -0,0 +1,319 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from collections import OrderedDict +from typing import Any, Dict, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchaudio.compliance.kaldi as Kaldi + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.models.audio.sv.DTDNN import CAMPPlus +from modelscope.utils.constant import Tasks + + +class MultiHeadSelfAttention(nn.Module): + + def __init__(self, n_units, h=8, dropout=0.1): + super(MultiHeadSelfAttention, self).__init__() + self.linearQ = nn.Linear(n_units, n_units) + self.linearK = nn.Linear(n_units, n_units) + self.linearV = nn.Linear(n_units, n_units) + self.linearO = nn.Linear(n_units, n_units) + self.d_k = n_units // h + self.h = h + self.dropout = nn.Dropout(p=dropout) + self.att = None + + def forward(self, x, batch_size): + # x: (BT, F) + q = self.linearQ(x).reshape(batch_size, -1, self.h, self.d_k) + k = self.linearK(x).reshape(batch_size, -1, self.h, self.d_k) + v = self.linearV(x).reshape(batch_size, -1, self.h, self.d_k) + scores = torch.matmul(q.transpose(1, 2), k.permute( + 0, 2, 3, 1)) / np.sqrt(self.d_k) + # scores: (B, h, T, T) + self.att = F.softmax(scores, dim=3) + p_att = self.dropout(self.att) + # v : (B, T, h, d_k) + # p_att : (B, h, T, T) + x = torch.matmul(p_att, v.transpose(1, 2)) + # x : (B, h, T, d_k) + x = x.transpose(1, 2).reshape(-1, self.h * self.d_k) + return self.linearO(x) + + +class PositionwiseFeedForward(nn.Module): + + def __init__(self, n_units, d_units, dropout): + super(PositionwiseFeedForward, self).__init__() + self.linear1 = nn.Linear(n_units, d_units) + self.linear2 = nn.Linear(d_units, n_units) + self.dropout = nn.Dropout(p=dropout) + + def forward(self, x): + return self.linear2(self.dropout(F.relu(self.linear1(x)))) + + +class PosEncoding(nn.Module): + + def __init__(self, max_seq_len, d_word_vec): + super(PosEncoding, self).__init__() + pos_enc = np.array([[ + pos / np.power(10000, 2.0 * (j // 2) / d_word_vec) + for j in range(d_word_vec) + ] for pos in range(max_seq_len)]) + pos_enc[:, 0::2] = np.sin(pos_enc[:, 0::2]) + pos_enc[:, 1::2] = np.cos(pos_enc[:, 1::2]) + pad_row = np.zeros([1, d_word_vec]) + pos_enc = np.concatenate([pad_row, pos_enc]).astype(np.float32) + + self.pos_enc = torch.nn.Embedding(max_seq_len + 1, d_word_vec) + self.pos_enc.weight = torch.nn.Parameter( + torch.from_numpy(pos_enc), requires_grad=False) + + def forward(self, input_len): + max_len = torch.max(input_len) + input_pos = torch.LongTensor([ + list(range(1, len + 1)) + [0] * (max_len - len) + for len in input_len + ]) + + return self.pos_enc(input_pos) + + +class TransformerEncoder(nn.Module): + + def __init__(self, + idim, + n_units=256, + n_layers=2, + e_units=512, + h=4, + dropout=0.1): + super(TransformerEncoder, self).__init__() + self.linear_in = nn.Linear(idim, n_units) + self.lnorm_in = nn.LayerNorm(n_units) + + self.n_layers = n_layers + self.dropout = nn.Dropout(p=dropout) + for i in range(n_layers): + setattr(self, '{}{:d}'.format('lnorm1_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('self_att_', i), + MultiHeadSelfAttention(n_units, h)) + setattr(self, '{}{:d}'.format('lnorm2_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('ff_', i), + PositionwiseFeedForward(n_units, e_units, dropout)) + self.lnorm_out = nn.LayerNorm(n_units) + + def forward(self, x): + # x: [B, num_anchors, T, n_in] + bs, num, tframe, dim = x.size() + x = x.reshape(bs * num, tframe, -1) # [B*num_anchors, T, dim] + # x: (B, T, F) ... batch, time, (mel)freq + B_size, T_size, _ = x.shape + # e: (BT, F) + e = self.linear_in(x.reshape(B_size * T_size, -1)) + # Encoder stack + for i in range(self.n_layers): + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm1_', i))(e) + # self-attention + s = getattr(self, '{}{:d}'.format('self_att_', i))(e, x.shape[0]) + # residual + e = e + self.dropout(s) + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm2_', i))(e) + # positionwise feed-forward + s = getattr(self, '{}{:d}'.format('ff_', i))(e) + # residual + e = e + self.dropout(s) + # final layer normalization + # output: (BT, F) + # output: (B, F, T) + output = self.lnorm_out(e).reshape(B_size, T_size, -1) + output = output.reshape(bs, num, tframe, + -1) # [B, num_anchors, T, dim] + return output + + +class TransformerEncoder_out(nn.Module): + + def __init__(self, + idim, + n_units=256, + n_layers=2, + e_units=512, + h=4, + dropout=0.1): + super(TransformerEncoder_out, self).__init__() + self.linear_in = nn.Linear(idim, n_units) + self.lnorm_in = nn.LayerNorm(n_units) + + self.n_layers = n_layers + self.dropout = nn.Dropout(p=dropout) + for i in range(n_layers): + setattr(self, '{}{:d}'.format('lnorm1_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('self_att_', i), + MultiHeadSelfAttention(n_units, h)) + setattr(self, '{}{:d}'.format('lnorm2_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('ff_', i), + PositionwiseFeedForward(n_units, e_units, dropout)) + self.lnorm_out = nn.LayerNorm(n_units) + + def forward(self, x): + # x: (B, T, F) + B_size, T_size, _ = x.shape + # e: (BT, F) + e = self.linear_in(x.reshape(B_size * T_size, -1)) + # Encoder stack + for i in range(self.n_layers): + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm1_', i))(e) + # self-attention + s = getattr(self, '{}{:d}'.format('self_att_', i))(e, x.shape[0]) + # residual + e = e + self.dropout(s) + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm2_', i))(e) + # positionwise feed-forward + s = getattr(self, '{}{:d}'.format('ff_', i))(e) + # residual + e = e + self.dropout(s) + # final layer normalization + # output: (BT, F) + # output: (B, T, F) + output = self.lnorm_out(e).reshape(B_size, T_size, -1) + return output + + +class OutLayer(nn.Module): + + def __init__(self, n_units=256, num_anchors=2): + super(OutLayer, self).__init__() + self.combine = TransformerEncoder_out(num_anchors * n_units, n_units) + self.out_linear = nn.Linear(n_units // num_anchors, 1) + + def forward(self, input): + # input: [B, num_anchors, T, dim] + bs, num, tframe, dim = input.size() + output = input.permute(0, 2, 1, + 3).reshape(bs, tframe, + -1) # [Bs, t, num_anchors*dim] + output = self.combine(output) # [Bs, t, n_units] + output = output.reshape( + bs, tframe, num, -1) # [Bs, t, num_anchors, n_units//num_anchors] + output = self.out_linear(output).squeeze(-1) # [Bs, t, num_anchors] + + return output + + +class TransformerDetector(nn.Module): + + def __init__(self, + frame_dim=512, + anchor_dim=192, + hidden_dim=256, + max_seq_len=1000): + super(TransformerDetector, self).__init__() + self.detection = TransformerEncoder( + idim=frame_dim + anchor_dim, n_units=hidden_dim) + self.output = OutLayer(n_units=hidden_dim) + self.pos_enc = PosEncoding(max_seq_len, hidden_dim) + + def forward(self, feats, anchors): + # feats: [1, t, fdim] + num_frames = feats.shape[1] + num_anchors = anchors.shape[1] + bs = feats.shape[0] + feats = feats.unsqueeze(1).repeat( + 1, num_anchors, 1, 1) # shape: [Bs, num_anchors, t, fdim] + anchors = anchors.unsqueeze(2).repeat( + 1, 1, num_frames, 1) # shape: [Bs, num_anchors, t, xdim] + sd_in = torch.cat((feats, anchors), + dim=-1) # shape: [Bs, num_anchors, t, fdim+xdim] + sd_out = self.detection(sd_in) # shape: [Bs, num_anchors, t, sd_dim] + + # pos + pos_emb = self.pos_enc(torch.tensor([num_frames] * (bs * num_anchors))) + pos_emb = pos_emb.reshape(bs, num_anchors, num_frames, -1) + sd_out += pos_emb + + # output + output = self.output(sd_out) # shape: [Bs, t, num_anchors] + + return output + + +@MODELS.register_module(Tasks.speaker_diarization, module_name=Models.scl_sd) +class SpeakerChangeLocatorTransformer(TorchModel): + r"""A speaekr change locator using the transformer architecture as the backbone. + Args: + model_dir: A model dir. + model_config: The model config. + """ + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + + self.feature_dim = self.model_config['fbank_dim'] + frame_size = self.model_config['frame_size'] + anchor_size = self.model_config['anchor_size'] + + self.encoder = CAMPPlus(self.feature_dim, output_level='frame') + self.backend = TransformerDetector( + frame_dim=frame_size, anchor_dim=anchor_size) + + pretrained_encoder = kwargs['pretrained_encoder'] + pretrained_backend = kwargs['pretrained_backend'] + + self.__load_check_point(pretrained_encoder, pretrained_backend) + + self.encoder.eval() + self.backend.eval() + + def forward(self, audio, anchors): + assert len(audio.shape) == 2 and audio.shape[ + 0] == 1, 'modelscope error: the shape of input audio to model needs to be [1, T]' + assert len( + anchors.shape + ) == 3 and anchors.shape[0] == 1 and anchors.shape[ + 1] == 2, 'modelscope error: the shape of input anchors to model needs to be [1, 2, D]' + # audio shape: [1, T] + feature = self.__extract_feature(audio) + frame_state = self.encoder(feature) + output = self.backend(frame_state, anchors) + output = output.squeeze(0).detach().cpu().sigmoid() + + time_scale_factor = int(np.ceil(feature.shape[1] / output.shape[0])) + output = output.unsqueeze(1).expand(-1, time_scale_factor, + -1).reshape(-1, output.shape[-1]) + return output + + def __extract_feature(self, audio): + feature = Kaldi.fbank(audio, num_mel_bins=self.feature_dim) + feature = feature - feature.mean(dim=0, keepdim=True) + feature = feature.unsqueeze(0) + return feature + + def __load_check_point(self, + pretrained_encoder, + pretrained_backend, + device=None): + if not device: + device = torch.device('cpu') + self.encoder.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_encoder), + map_location=device)) + + self.backend.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_backend), + map_location=device)) diff --git a/modelscope/pipelines/audio/speaker_change_locating_pipeline.py b/modelscope/pipelines/audio/speaker_change_locating_pipeline.py new file mode 100644 index 00000000..0bab08ac --- /dev/null +++ b/modelscope/pipelines/audio/speaker_change_locating_pipeline.py @@ -0,0 +1,105 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import io +from typing import Any, Dict, List, Union + +import numpy as np +import soundfile as sf +import torch + +from modelscope.fileio import File +from modelscope.metainfo import Pipelines +from modelscope.outputs import OutputKeys +from modelscope.pipelines.base import InputModel, Pipeline +from modelscope.pipelines.builder import PIPELINES +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['SpeakerChangeLocatingPipeline'] + + +@PIPELINES.register_module( + Tasks.speaker_diarization, module_name=Pipelines.speaker_change_locating) +class SpeakerChangeLocatingPipeline(Pipeline): + """Speaker Change Locating Inference Pipeline + use `model` to create a speaker change Locating pipeline. + + Args: + model (SpeakerChangeLocatingPipeline): A model instance, or a model local dir, or a model id in the model hub. + kwargs (dict, `optional`): + Extra kwargs passed into the pipeline's constructor. + Example: + >>> from modelscope.pipelines import pipeline + >>> from modelscope.utils.constant import Tasks + >>> p = pipeline( + >>> task=Tasks.speaker_diarization, model='damo/speech_campplus-transformer_scl_zh-cn_16k-common') + >>> print(p(audio)) + + """ + + def __init__(self, model: InputModel, **kwargs): + """use `model` to create a speaker change Locating pipeline for prediction + Args: + model (str): a valid offical model id + """ + super().__init__(model=model, **kwargs) + self.model_config = self.model.model_config + self.config = self.model.model_config + self.anchor_size = self.config['anchor_size'] + + def __call__(self, audio: str, embds: List = None) -> Dict[str, Any]: + if embds is not None: + assert len(embds) == 2 + assert isinstance(embds[0], np.ndarray) and isinstance( + embds[1], np.ndarray) + assert embds[0].shape == ( + self.anchor_size, ) and embds[1].shape == (self.anchor_size, ) + else: + embd1 = np.zeros(self.anchor_size // 2) + embd2 = np.ones(self.anchor_size - self.anchor_size // 2) + embd3 = np.ones(self.anchor_size // 2) + embd4 = np.zeros(self.anchor_size - self.anchor_size // 2) + embds = [ + np.stack([embd1, embd2], axis=1).flatten(), + np.stack([embd3, embd4], axis=1).flatten(), + ] + anchors = torch.from_numpy(np.stack(embds, + axis=0)).float().unsqueeze(0) + + output = self.preprocess(audio) + output = self.forward(output, anchors) + output = self.postprocess(output) + + return output + + def forward(self, input: torch.Tensor, anchors: torch.Tensor): + output = self.model(input, anchors) + return output + + def postprocess(self, input: torch.Tensor) -> Dict[str, Any]: + predict = np.where(np.diff(input.argmax(-1).numpy())) + try: + predict = predict[0][0] * 0.01 + 0.02 + predict = round(predict, 2) + return {OutputKeys.TEXT: f'The change point is at {predict}s.'} + except Exception: + return {OutputKeys.TEXT: 'No change point is found.'} + + def preprocess(self, input: str) -> torch.Tensor: + if isinstance(input, str): + file_bytes = File.read(input) + data, fs = sf.read(io.BytesIO(file_bytes), dtype='float32') + if len(data.shape) == 2: + data = data[:, 0] + if fs != self.model_config['sample_rate']: + raise ValueError( + 'modelscope error: Only support %d sample rate files' + % self.model_cfg['sample_rate']) + data = torch.from_numpy(data).unsqueeze(0) + else: + raise ValueError( + 'modelscope error: The input type is restricted to audio file address' + % i) + return data diff --git a/tests/pipelines/test_speaker_verification.py b/tests/pipelines/test_speaker_verification.py index 236b952b..41335fc8 100644 --- a/tests/pipelines/test_speaker_verification.py +++ b/tests/pipelines/test_speaker_verification.py @@ -2,7 +2,7 @@ import os.path import unittest -from typing import Any, Dict, List +from typing import Any, Dict, List, Union from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline @@ -16,20 +16,25 @@ logger = get_logger() SPEAKER1_A_EN_16K_WAV = 'data/test/audios/speaker1_a_en_16k.wav' SPEAKER1_B_EN_16K_WAV = 'data/test/audios/speaker1_b_en_16k.wav' SPEAKER2_A_EN_16K_WAV = 'data/test/audios/speaker2_a_en_16k.wav' +SCL_EXAMPLE_WAV = 'data/test/audios/scl_example1.wav' class SpeakerVerificationTest(unittest.TestCase, DemoCompatibilityCheck): ecapatdnn_voxceleb_16k_model_id = 'damo/speech_ecapa-tdnn_sv_en_voxceleb_16k' campplus_voxceleb_16k_model_id = 'damo/speech_campplus_sv_en_voxceleb_16k' rdino_voxceleb_16k_model_id = 'damo/speech_rdino_ecapa_tdnn_sv_en_voxceleb_16k' + speaker_change_locating_cn_model_id = 'damo/speech_campplus-transformer_scl_zh-cn_16k-common' def setUp(self) -> None: self.task = Tasks.speaker_verification def run_pipeline(self, model_id: str, - audios: List[str], + audios: Union[List[str], str], + task: str = None, model_revision=None) -> Dict[str, Any]: + if task is not None: + self.task = task p = pipeline( task=self.task, model=model_id, model_revision=model_revision) result = p(audios) @@ -66,6 +71,17 @@ class SpeakerVerificationTest(unittest.TestCase, DemoCompatibilityCheck): print(result) self.assertTrue(OutputKeys.SCORE in result) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_speaker_change_locating_cn_16k(self): + logger.info( + 'Run speaker change locating for campplus-transformer model') + result = self.run_pipeline( + model_id=self.speaker_change_locating_cn_model_id, + task=Tasks.speaker_diarization, + audios=SCL_EXAMPLE_WAV) + print(result) + self.assertTrue(OutputKeys.TEXT in result) + @unittest.skip('demo compatibility test is only enabled on a needed-basis') def test_demo_compatibility(self): self.compatibility_check() From 7fdfb1cd5d78c6125096882b74ed76f45b3de84b Mon Sep 17 00:00:00 2001 From: "xingjun.wang" Date: Fri, 12 May 2023 19:05:12 +0800 Subject: [PATCH 32/76] change version --- modelscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelscope/version.py b/modelscope/version.py index 81c35379..02464c29 100644 --- a/modelscope/version.py +++ b/modelscope/version.py @@ -1,5 +1,5 @@ # Make sure to modify __release_datetime__ to release time when making official release. -__version__ = '1.5.0' +__version__ = '1.6.0' # default release datetime for branches under active development is set # to be a time far-far-away-into-the-future __release_datetime__ = '2099-10-13 08:56:12' From 4d5b111b08baa3b81458ae69b44ef3d2dc7eb94b Mon Sep 17 00:00:00 2001 From: "xingjun.wang" Date: Fri, 12 May 2023 19:54:16 +0800 Subject: [PATCH 33/76] update readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 3bed2b4a..c9b071ab 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,10 @@ Audio: * [u2pp_conformer-asr-cn-16k-online](https://modelscope.cn/models/wenet/u2pp_conformer-asr-cn-16k-online) +* [speech_fsmn_vad_zh-cn-16k-common-pytorch](https://modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-pytorch/summary) + +* [punc_ct-transformer_zh-cn-common-vocab272727-pytorch](https://modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/summary) + * [speech_frcrn_ans_cirm_16k](https://modelscope.cn/models/damo/speech_frcrn_ans_cirm_16k) * [speech_dfsmn_aec_psm_16k](https://modelscope.cn/models/damo/speech_dfsmn_aec_psm_16k) From febc0365de294ab08fcfa1768b929473f6e47426 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Sat, 13 May 2023 12:12:04 +0800 Subject: [PATCH 34/76] Support FlexTrain and update the structure of trainer 1. Refactor training_args 2. Refactor hooks 3. Add train_id for push_to_hub 4. Support both output_dir/output_sub_dir for checkpoint_hooks 5. Support copy when hardlink fails when checkpointing 6. Support mixed dataset config file as a CLI argument 7. Add eval txt in output folder Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12384253 * support the ignorance of file pattern --- .../finetune_image_classification.py | 2 +- .../finetune_multi_modal_embedding.py | 4 +- .../finetune_text_classification.py | 57 +- .../pytorch/text_classification/run_train.sh | 5 +- .../finetune_text_generation.py | 48 +- .../pytorch/text_generation/run_train_gpt3.sh | 5 +- .../finetune_token_classification.py | 4 +- modelscope/__init__.py | 79 +- modelscope/hub/api.py | 19 +- modelscope/hub/file_download.py | 1 - modelscope/hub/push_to_hub.py | 44 +- modelscope/models/audio/tts/voice.py | 4 +- modelscope/pipelines/__init__.py | 2 - modelscope/preprocessors/__init__.py | 2 +- modelscope/trainers/__init__.py | 6 +- modelscope/trainers/cli_argument_parser.py | 151 ++ modelscope/trainers/default_config.py | 37 +- modelscope/trainers/hooks/__init__.py | 6 +- .../trainers/hooks/checkpoint/__init__.py | 2 + .../hooks/checkpoint/checkpoint_hook.py | 435 ++++++ .../hooks/checkpoint/checkpoint_processor.py | 276 ++++ .../hooks/checkpoint/load_checkpoint_hook.py | 138 ++ modelscope/trainers/hooks/checkpoint_hook.py | 749 ---------- .../hooks/compression/sparsity_hook.py | 1 - .../trainers/hooks/distributed/__init__.py | 3 + .../hooks/{ => distributed}/ddp_hook.py | 6 +- .../hooks/{ => distributed}/deepspeed_hook.py | 145 +- .../hooks/{ => distributed}/megatron_hook.py | 239 +-- modelscope/trainers/hooks/early_stop_hook.py | 30 +- modelscope/trainers/hooks/evaluation_hook.py | 30 +- modelscope/trainers/hooks/hook.py | 72 +- .../trainers/hooks/lr_scheduler_hook.py | 132 +- .../hooks/optimizer/apex_optimizer_hook.py | 72 +- modelscope/trainers/hooks/optimizer/base.py | 89 +- .../hooks/optimizer/torch_optimizer_hook.py | 84 +- modelscope/trainers/trainer.py | 72 +- modelscope/trainers/training_args.py | 1315 +++++++---------- modelscope/utils/ast_utils.py | 6 +- modelscope/utils/checkpoint.py | 3 +- tests/hub/test_hub_upload.py | 1 + tests/trainers/cli/__init__.py | 0 tests/trainers/cli/test_cli.py | 52 + .../trainers/hooks/test_lr_scheduler_hook.py | 3 + tests/trainers/hooks/test_optimizer_hook.py | 1 + tests/trainers/test_trainer_with_nlp.py | 53 +- tests/trainers/test_training_args.py | 10 +- 46 files changed, 2394 insertions(+), 2101 deletions(-) create mode 100644 modelscope/trainers/cli_argument_parser.py create mode 100644 modelscope/trainers/hooks/checkpoint/__init__.py create mode 100644 modelscope/trainers/hooks/checkpoint/checkpoint_hook.py create mode 100644 modelscope/trainers/hooks/checkpoint/checkpoint_processor.py create mode 100644 modelscope/trainers/hooks/checkpoint/load_checkpoint_hook.py delete mode 100644 modelscope/trainers/hooks/checkpoint_hook.py create mode 100644 modelscope/trainers/hooks/distributed/__init__.py rename modelscope/trainers/hooks/{ => distributed}/ddp_hook.py (89%) rename modelscope/trainers/hooks/{ => distributed}/deepspeed_hook.py (64%) rename modelscope/trainers/hooks/{ => distributed}/megatron_hook.py (70%) create mode 100644 tests/trainers/cli/__init__.py create mode 100644 tests/trainers/cli/test_cli.py diff --git a/examples/pytorch/image_classification/finetune_image_classification.py b/examples/pytorch/image_classification/finetune_image_classification.py index 4e96c2cd..1e7fceb1 100644 --- a/examples/pytorch/image_classification/finetune_image_classification.py +++ b/examples/pytorch/image_classification/finetune_image_classification.py @@ -3,8 +3,8 @@ from dataclasses import dataclass, field from modelscope.metainfo import Trainers from modelscope.msdatasets.ms_dataset import MsDataset +from modelscope.trainers.args import TrainingArgs from modelscope.trainers.builder import build_trainer -from modelscope.trainers.training_args import TrainingArgs @dataclass diff --git a/examples/pytorch/multi_modal_embedding/finetune_multi_modal_embedding.py b/examples/pytorch/multi_modal_embedding/finetune_multi_modal_embedding.py index cc7da842..b3c325cf 100644 --- a/examples/pytorch/multi_modal_embedding/finetune_multi_modal_embedding.py +++ b/examples/pytorch/multi_modal_embedding/finetune_multi_modal_embedding.py @@ -5,8 +5,8 @@ from functools import partial from modelscope.metainfo import Trainers from modelscope.msdatasets import MsDataset from modelscope.trainers import build_trainer -from modelscope.trainers.training_args import (TrainingArgs, get_flatten_value, - set_flatten_value) +from modelscope.trainers.args import (TrainingArgs, get_flatten_value, + set_flatten_value) @dataclass diff --git a/examples/pytorch/text_classification/finetune_text_classification.py b/examples/pytorch/text_classification/finetune_text_classification.py index 7747bc25..73d15783 100644 --- a/examples/pytorch/text_classification/finetune_text_classification.py +++ b/examples/pytorch/text_classification/finetune_text_classification.py @@ -1,26 +1,17 @@ import os from dataclasses import dataclass, field -from modelscope.msdatasets import MsDataset -from modelscope.trainers import EpochBasedTrainer, build_trainer -from modelscope.trainers.training_args import TrainingArgs +from modelscope import (EpochBasedTrainer, MsDataset, TrainingArgs, + build_dataset_from_file, build_trainer) -def get_labels(cfg, metadata): - label2id = cfg.safe_get(metadata['cfg_node']) - if label2id is not None: - return ','.join(label2id.keys()) - - -def set_labels(cfg, labels, metadata): +def set_labels(labels): if isinstance(labels, str): labels = labels.split(',') - cfg.merge_from_dict( - {metadata['cfg_node']: {label: id - for id, label in enumerate(labels)}}) + return {label: id for id, label in enumerate(labels)} -@dataclass +@dataclass(init=False) class TextClassificationArguments(TrainingArgs): first_sequence: str = field( @@ -49,7 +40,6 @@ class TextClassificationArguments(TrainingArgs): metadata={ 'help': 'The labels of the dataset', 'cfg_node': 'preprocessor.label2id', - 'cfg_getter': get_labels, 'cfg_setter': set_labels, }) @@ -60,30 +50,39 @@ class TextClassificationArguments(TrainingArgs): 'cfg_node': 'preprocessor.type' }) - def __call__(self, config): - config = super().__call__(config) - config.model['num_labels'] = len(self.labels) - if config.train.lr_scheduler.type == 'LinearLR': - config.train.lr_scheduler['total_iters'] = \ - int(len(train_dataset) / self.per_device_train_batch_size) * self.max_epochs - return config + +config, args = TextClassificationArguments().parse_cli().to_config() + +print(config, args) -args = TextClassificationArguments.from_cli( - task='text-classification', eval_metrics='seq-cls-metric') +def cfg_modify_fn(cfg): + if args.use_model_config: + cfg.merge_from_dict(config) + else: + cfg = config + cfg.model['num_labels'] = len(cfg.preprocessor.label2id) + if cfg.train.lr_scheduler.type == 'LinearLR': + cfg.train.lr_scheduler['total_iters'] = \ + int(len(train_dataset) / cfg.train.dataloader.batch_size_per_gpu) * cfg.train.max_epochs + return cfg -print(args) -dataset = MsDataset.load(args.dataset_name, subset_name=args.subset_name) -train_dataset = dataset['train'] -validation_dataset = dataset['validation'] +if args.dataset_json_file is None: + dataset = MsDataset.load( + args.train_dataset_name, subset_name=args.train_subset_name) + train_dataset = dataset['train'] + validation_dataset = dataset['validation'] +else: + train_dataset, validation_dataset = build_dataset_from_file( + args.dataset_json_file) kwargs = dict( model=args.model, train_dataset=train_dataset, eval_dataset=validation_dataset, seed=args.seed, - cfg_modify_fn=args) + cfg_modify_fn=cfg_modify_fn) os.environ['LOCAL_RANK'] = str(args.local_rank) trainer: EpochBasedTrainer = build_trainer(name='trainer', default_args=kwargs) diff --git a/examples/pytorch/text_classification/run_train.sh b/examples/pytorch/text_classification/run_train.sh index 93c23d0d..f46fb72a 100644 --- a/examples/pytorch/text_classification/run_train.sh +++ b/examples/pytorch/text_classification/run_train.sh @@ -1,7 +1,7 @@ PYTHONPATH=. python examples/pytorch/text_classification/finetune_text_classification.py \ --model 'damo/nlp_structbert_backbone_base_std' \ - --dataset_name 'clue' \ - --subset_name 'tnews' \ + --train_dataset_name 'clue' \ + --train_subset_name 'tnews' \ --first_sequence 'sentence' \ --preprocessor.label label \ --model.num_labels 15 \ @@ -10,3 +10,4 @@ PYTHONPATH=. python examples/pytorch/text_classification/finetune_text_classific --train.dataloader.workers_per_gpu 0 \ --evaluation.dataloader.workers_per_gpu 0 \ --train.optimizer.lr 1e-5 \ + --use_model_config true \ diff --git a/examples/pytorch/text_generation/finetune_text_generation.py b/examples/pytorch/text_generation/finetune_text_generation.py index 7a140a0c..f9e47bf3 100644 --- a/examples/pytorch/text_generation/finetune_text_generation.py +++ b/examples/pytorch/text_generation/finetune_text_generation.py @@ -1,12 +1,11 @@ from dataclasses import dataclass, field +from modelscope import (EpochBasedTrainer, MsDataset, TrainingArgs, + build_trainer) from modelscope.metainfo import Trainers -from modelscope.msdatasets import MsDataset -from modelscope.trainers import EpochBasedTrainer, build_trainer -from modelscope.trainers.training_args import TrainingArgs -@dataclass +@dataclass(init=False) class TextGenerationArguments(TrainingArgs): trainer: str = field( @@ -67,30 +66,35 @@ class TextGenerationArguments(TrainingArgs): 'help': 'Whether to use MegatronHook', }) - def __call__(self, config): - config = super().__call__(config) - if config.train.lr_scheduler.type == 'noam': - config.train.lr_scheduler = { - 'type': 'LambdaLR', - 'lr_lambda': noam_lambda, - 'options': { - 'by_epoch': False - } - } - if self.use_megatron: - config.train.hooks.append({'type': 'MegatronHook'}) - return config - def noam_lambda(current_step: int): current_step += 1 return min(current_step**(-0.5), current_step * 100**(-1.5)) -args = TextGenerationArguments.from_cli(task='text-generation') -print(args) +config, args = TextGenerationArguments().parse_cli().to_config() +print(config, args) -dataset = MsDataset.load(args.dataset_name) + +def cfg_modify_fn(cfg): + if args.use_model_config: + cfg.merge_from_dict(config) + else: + cfg = config + if cfg.train.lr_scheduler.type == 'noam': + cfg.train.lr_scheduler = { + 'type': 'LambdaLR', + 'lr_lambda': noam_lambda, + 'options': { + 'by_epoch': False + } + } + if args.use_megatron: + cfg.train.hooks.append({'type': 'MegatronHook'}) + return cfg + + +dataset = MsDataset.load(args.train_dataset_name) train_dataset = dataset['train'] eval_dataset = dataset['validation' if 'validation' in dataset else 'test'] @@ -100,7 +104,7 @@ kwargs = dict( eval_dataset=eval_dataset, seed=args.seed, work_dir=args.work_dir, - cfg_modify_fn=args) + cfg_modify_fn=cfg_modify_fn) trainer: EpochBasedTrainer = build_trainer( name=args.trainer, default_args=kwargs) diff --git a/examples/pytorch/text_generation/run_train_gpt3.sh b/examples/pytorch/text_generation/run_train_gpt3.sh index a20a5bb2..fd37b42c 100644 --- a/examples/pytorch/text_generation/run_train_gpt3.sh +++ b/examples/pytorch/text_generation/run_train_gpt3.sh @@ -8,7 +8,7 @@ PYTHONPATH=. torchrun --nproc_per_node $WORLD_SIZE examples/pytorch/text_generat --trainer 'nlp-gpt3-trainer' \ --work_dir './tmp' \ --model 'damo/nlp_gpt3_text-generation_1.3B' \ - --dataset_name 'chinese-poetry-collection' \ + --train_dataset_name 'chinese-poetry-collection' \ --preprocessor 'text-gen-jieba-tokenizer' \ --src_txt 'text1' \ --tgt_txt 'text2' \ @@ -20,4 +20,5 @@ PYTHONPATH=. torchrun --nproc_per_node $WORLD_SIZE examples/pytorch/text_generat --world_size $WORLD_SIZE \ --tensor_model_parallel_size $TENSOR_MODEL_PARALLEL_SIZE \ --use_megatron true \ - # --dataset_name 'DuReader_robust-QG' \ # input&output + --use_model_config true \ + # --train_dataset_name 'DuReader_robust-QG' \ # input&output diff --git a/examples/pytorch/token_classification/finetune_token_classification.py b/examples/pytorch/token_classification/finetune_token_classification.py index cf51ed22..e06f6f48 100644 --- a/examples/pytorch/token_classification/finetune_token_classification.py +++ b/examples/pytorch/token_classification/finetune_token_classification.py @@ -3,8 +3,8 @@ from dataclasses import dataclass, field from modelscope.metainfo import Trainers from modelscope.msdatasets import MsDataset from modelscope.trainers import build_trainer -from modelscope.trainers.training_args import (TrainingArgs, get_flatten_value, - set_flatten_value) +from modelscope.trainers.args import (TrainingArgs, get_flatten_value, + set_flatten_value) @dataclass diff --git a/modelscope/__init__.py b/modelscope/__init__.py index 81fdf505..f7553958 100644 --- a/modelscope/__init__.py +++ b/modelscope/__init__.py @@ -1,4 +1,79 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -from .version import __release_datetime__, __version__ +from typing import TYPE_CHECKING -__all__ = ['__version__', '__release_datetime__'] +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .version import __release_datetime__, __version__ + from .trainers import EpochBasedTrainer, TrainingArgs, build_dataset_from_file + from .trainers import Hook, Priority + from .exporters import Exporter + from .exporters import TfModelExporter + from .exporters import TorchModelExporter + from .hub.api import HubApi + from .hub.snapshot_download import snapshot_download + from .hub.push_to_hub import push_to_hub, push_to_hub_async + from .hub.check_model import check_model_is_id, check_local_model_is_latest + from .metrics import AudioNoiseMetric, Metric, task_default_metrics, ImageColorEnhanceMetric, ImageDenoiseMetric, \ + ImageInstanceSegmentationCOCOMetric, ImagePortraitEnhancementMetric, SequenceClassificationMetric, \ + TextGenerationMetric, TokenClassificationMetric, VideoSummarizationMetric, MovieSceneSegmentationMetric, \ + AccuracyMetric, BleuMetric, ImageInpaintingMetric, ReferringVideoObjectSegmentationMetric, \ + VideoFrameInterpolationMetric, VideoStabilizationMetric, VideoSuperResolutionMetric, PplMetric, \ + ImageQualityAssessmentDegradationMetric, ImageQualityAssessmentMosMetric, TextRankingMetric, \ + LossMetric, ImageColorizationMetric, OCRRecognitionMetric + from .models import Model, TorchModel + from .preprocessors import Preprocessor + from .pipelines import Pipeline, pipeline + from .utils.hub import read_config, create_model_if_not_exist + from .utils.logger import get_logger + from .msdatasets import MsDataset + +else: + _import_structure = { + 'version': ['__release_datetime__', '__version__'], + 'trainers': [ + 'EpochBasedTrainer', 'TrainingArgs', 'Hook', 'Priority', + 'build_dataset_from_file' + ], + 'exporters': [ + 'Exporter', + 'TfModelExporter', + 'TorchModelExporter', + ], + 'hub.api': ['HubApi'], + 'hub.snapshot_download': ['snapshot_download'], + 'hub.push_to_hub': ['push_to_hub', 'push_to_hub_async'], + 'hub.check_model': + ['check_model_is_id', 'check_local_model_is_latest'], + 'metrics': [ + 'AudioNoiseMetric', 'Metric', 'task_default_metrics', + 'ImageColorEnhanceMetric', 'ImageDenoiseMetric', + 'ImageInstanceSegmentationCOCOMetric', + 'ImagePortraitEnhancementMetric', 'SequenceClassificationMetric', + 'TextGenerationMetric', 'TokenClassificationMetric', + 'VideoSummarizationMetric', 'MovieSceneSegmentationMetric', + 'AccuracyMetric', 'BleuMetric', 'ImageInpaintingMetric', + 'ReferringVideoObjectSegmentationMetric', + 'VideoFrameInterpolationMetric', 'VideoStabilizationMetric', + 'VideoSuperResolutionMetric', 'PplMetric', + 'ImageQualityAssessmentDegradationMetric', + 'ImageQualityAssessmentMosMetric', 'TextRankingMetric', + 'LossMetric', 'ImageColorizationMetric', 'OCRRecognitionMetric' + ], + 'models': ['Model', 'TorchModel'], + 'preprocessors': ['Preprocessor'], + 'pipelines': ['Pipeline', 'pipeline'], + 'utils.hub': ['read_config', 'create_model_if_not_exist'], + 'utils.logger': ['get_logger'], + 'msdatasets': ['MsDataset'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 0991dbb9..e3436aea 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -6,6 +6,7 @@ import functools import os import pickle import platform +import re import shutil import tempfile import uuid @@ -19,7 +20,6 @@ import requests from requests import Session from requests.adapters import HTTPAdapter, Retry -from modelscope import __version__ from modelscope.hub.constants import (API_HTTP_CLIENT_TIMEOUT, API_RESPONSE_FIELD_DATA, API_RESPONSE_FIELD_EMAIL, @@ -161,6 +161,7 @@ class HubApi: 'Visibility': visibility, # server check 'License': license, 'OriginalModelId': original_model_id, + 'TrainId': os.environ.get('MODELSCOPE_TRAIN_ID', ''), } r = self.session.post( path, json=body, cookies=cookies, headers=self.headers) @@ -237,8 +238,10 @@ class HubApi: license: Optional[str] = Licenses.APACHE_V2, chinese_name: Optional[str] = None, commit_message: Optional[str] = 'upload model', + tag: Optional[str] = None, revision: Optional[str] = DEFAULT_REPOSITORY_REVISION, - original_model_id: Optional[str] = None): + original_model_id: Optional[str] = None, + ignore_file_pattern: Optional[Union[List[str], str]] = None): """Upload model from a given directory to given repository. A valid model directory must contain a configuration.json file. @@ -269,10 +272,13 @@ class HubApi: chinese name of the new created model. commit_message(`str`, *optional*, defaults to `None`): commit message of the push request. + tag(`str`, *optional*, defaults to `None`): + The tag on this commit revision (`str`, *optional*, default to DEFAULT_MODEL_REVISION): which branch to push. If the branch is not exists, It will create a new branch and push to it. original_model_id (str, optional): The base model id which this model is trained from + ignore_file_pattern (`Union[List[str], str]`, optional): The file pattern to ignore uploading Raises: InvalidParameter: Parameter invalid. @@ -293,6 +299,10 @@ class HubApi: if cookies is None: raise NotLoginException('Must login before upload!') files_to_save = os.listdir(model_dir) + if ignore_file_pattern is None: + ignore_file_pattern = [] + if isinstance(ignore_file_pattern, str): + ignore_file_pattern = [ignore_file_pattern] try: self.get_model(model_id=model_id) except Exception: @@ -326,6 +336,8 @@ class HubApi: shutil.rmtree(src, ignore_errors=True) for f in files_to_save: if f[0] != '.': + if any([re.search(pattern, f) is not None for pattern in ignore_file_pattern]): + continue src = os.path.join(model_dir, f) if os.path.isdir(src): shutil.copytree(src, os.path.join(tmp_dir, f)) @@ -339,6 +351,8 @@ class HubApi: commit_message=commit_message, local_branch=revision, remote_branch=revision) + if tag is not None: + repo.tag_and_push(tag, tag) except Exception: raise finally: @@ -928,6 +942,7 @@ class ModelScopeConfig: if MODELSCOPE_CLOUD_USERNAME in os.environ: user_name = os.environ[MODELSCOPE_CLOUD_USERNAME] + from modelscope import __version__ ua = 'modelscope/%s; python/%s; session_id/%s; platform/%s; processor/%s; env/%s; user/%s' % ( __version__, platform.python_version(), diff --git a/modelscope/hub/file_download.py b/modelscope/hub/file_download.py index 380d2432..6d3ad63d 100644 --- a/modelscope/hub/file_download.py +++ b/modelscope/hub/file_download.py @@ -12,7 +12,6 @@ import requests from requests.adapters import Retry from tqdm import tqdm -from modelscope import __version__ from modelscope.hub.api import HubApi, ModelScopeConfig from modelscope.hub.constants import (API_FILE_DOWNLOAD_CHUNK_SIZE, API_FILE_DOWNLOAD_RETRY_TIMES, diff --git a/modelscope/hub/push_to_hub.py b/modelscope/hub/push_to_hub.py index ee7b240e..d117cc7f 100644 --- a/modelscope/hub/push_to_hub.py +++ b/modelscope/hub/push_to_hub.py @@ -4,8 +4,8 @@ import concurrent.futures import os from modelscope.hub.api import HubApi -from modelscope.hub.constants import Licenses, ModelVisibility -from modelscope.hub.errors import NotExistError +from modelscope.hub.constants import ModelVisibility +from modelscope.utils.constant import DEFAULT_REPOSITORY_REVISION from modelscope.utils.logger import get_logger logger = get_logger() @@ -18,7 +18,10 @@ def _api_push_to_hub(repo_name, token, private=True, commit_message='', - source_repo=''): + tag=None, + source_repo='', + ignore_file_pattern=None, + revision=DEFAULT_REPOSITORY_REVISION): try: api = HubApi() api.login(token) @@ -29,7 +32,10 @@ def _api_push_to_hub(repo_name, if not private else ModelVisibility.PRIVATE, chinese_name=repo_name, commit_message=commit_message, - original_model_id=source_repo) + tag=tag, + original_model_id=source_repo, + ignore_file_pattern=ignore_file_pattern, + revision=revision) commit_message = commit_message or 'No commit message' logger.info( f'Successfully upload the model to {repo_name} with message: {commit_message}' @@ -48,7 +54,10 @@ def push_to_hub(repo_name, private=True, retry=3, commit_message='', - source_repo=''): + tag=None, + source_repo='', + ignore_file_pattern=None, + revision=DEFAULT_REPOSITORY_REVISION): """ Args: repo_name: The repo name for the modelhub repo @@ -57,13 +66,18 @@ def push_to_hub(repo_name, private: If is a private repo, default True retry: Retry times if something error in uploading, default 3 commit_message: The commit message + tag: The tag of this commit source_repo: The source repo (model id) which this model comes from - + ignore_file_pattern: The file pattern to be ignored in uploading. + revision: The branch to commit to Returns: The boolean value to represent whether the model is uploaded. """ if token is None: token = os.environ.get('MODELSCOPE_API_TOKEN') + if ignore_file_pattern is None: + ignore_file_pattern = os.environ.get('UPLOAD_IGNORE_FILE_PATTERN') + assert repo_name is not None assert token is not None, 'Either pass in a token or to set `MODELSCOPE_API_TOKEN` in the environment variables.' assert os.path.isdir(output_dir) assert 'configuration.json' in os.listdir(output_dir) or 'configuration.yaml' in os.listdir(output_dir) \ @@ -73,7 +87,8 @@ def push_to_hub(repo_name, f'Uploading {output_dir} to {repo_name} with message {commit_message}') for i in range(retry): if _api_push_to_hub(repo_name, output_dir, token, private, - commit_message, source_repo): + commit_message, tag, source_repo, + ignore_file_pattern, revision): return True return False @@ -83,7 +98,10 @@ def push_to_hub_async(repo_name, token=None, private=True, commit_message='', - source_repo=''): + tag=None, + source_repo='', + ignore_file_pattern=None, + revision=DEFAULT_REPOSITORY_REVISION): """ Args: repo_name: The repo name for the modelhub repo @@ -91,13 +109,18 @@ def push_to_hub_async(repo_name, token: The user api token, function will check the `MODELSCOPE_API_TOKEN` variable if this argument is None private: If is a private repo, default True commit_message: The commit message + tag: The tag of this commit source_repo: The source repo (model id) which this model comes from - + ignore_file_pattern: The file pattern to be ignored in uploading + revision: The branch to commit to Returns: A handler to check the result and the status """ if token is None: token = os.environ.get('MODELSCOPE_API_TOKEN') + if ignore_file_pattern is None: + ignore_file_pattern = os.environ.get('UPLOAD_IGNORE_FILE_PATTERN') + assert repo_name is not None assert token is not None, 'Either pass in a token or to set `MODELSCOPE_API_TOKEN` in the environment variables.' assert os.path.isdir(output_dir) assert 'configuration.json' in os.listdir(output_dir) or 'configuration.yaml' in os.listdir(output_dir) \ @@ -106,4 +129,5 @@ def push_to_hub_async(repo_name, logger.info( f'Uploading {output_dir} to {repo_name} with message {commit_message}') return _executor.submit(_api_push_to_hub, repo_name, output_dir, token, - private, commit_message, source_repo) + private, commit_message, tag, source_repo, + ignore_file_pattern, revision) diff --git a/modelscope/models/audio/tts/voice.py b/modelscope/models/audio/tts/voice.py index 645a528f..ed9edf43 100644 --- a/modelscope/models/audio/tts/voice.py +++ b/modelscope/models/audio/tts/voice.py @@ -17,11 +17,9 @@ from kantts.train.trainer import GAN_Trainer, Sambert_Trainer, distributed_init from kantts.utils.ling_unit.ling_unit import KanTtsLinguisticUnit from torch.utils.data import DataLoader -from modelscope import __version__ from modelscope.utils.audio.audio_utils import TtsCustomParams from modelscope.utils.audio.tts_exceptions import ( TtsModelConfigurationException, TtsModelNotExistsException) -from modelscope.utils.constant import ModelFile, Tasks from modelscope.utils.logger import get_logger logger = get_logger() @@ -394,6 +392,7 @@ class Voice: logger.info(f'TRAINING steps: {train_max_steps}') config['create_time'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) + from modelscope import __version__ config['modelscope_version'] = __version__ with open(os.path.join(stage_dir, 'config.yaml'), 'w') as f: @@ -558,6 +557,7 @@ class Voice: logger.info(f'resume from: {resume_from}') config['create_time'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) + from modelscope import __version__ config['modelscope_version'] = __version__ with open(os.path.join(stage_dir, 'config.yaml'), 'w') as f: diff --git a/modelscope/pipelines/__init__.py b/modelscope/pipelines/__init__.py index 71fe307b..d98a7af9 100644 --- a/modelscope/pipelines/__init__.py +++ b/modelscope/pipelines/__init__.py @@ -1,7 +1,5 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -from typing import TYPE_CHECKING -from modelscope.utils.import_utils import LazyImportModule from . import audio, cv, multi_modal, nlp from .base import Pipeline from .builder import pipeline diff --git a/modelscope/preprocessors/__init__.py b/modelscope/preprocessors/__init__.py index fab055db..3bbca124 100644 --- a/modelscope/preprocessors/__init__.py +++ b/modelscope/preprocessors/__init__.py @@ -35,7 +35,7 @@ if TYPE_CHECKING: TextErrorCorrectionPreprocessor, TextGenerationT5Preprocessor, WordAlignmentPreprocessor, TextGenerationTransformersPreprocessor, Tokenize, WordSegmentationBlankSetToLabelPreprocessor, - CodeGeeXPreprocessor, MGLMSummarizationPreprocessor, + MGLMSummarizationPreprocessor, ZeroShotClassificationTransformersPreprocessor, TextGenerationJiebaPreprocessor, SentencePiecePreprocessor, DialogIntentPredictionPreprocessor, DialogModelingPreprocessor, diff --git a/modelscope/trainers/__init__.py b/modelscope/trainers/__init__.py index 90f73a7f..0d20fe00 100644 --- a/modelscope/trainers/__init__.py +++ b/modelscope/trainers/__init__.py @@ -15,6 +15,8 @@ if TYPE_CHECKING: from .nlp import SequenceClassificationTrainer, TextRankingTrainer, SiameseUIETrainer from .nlp_trainer import NlpEpochBasedTrainer, VecoTrainer from .trainer import EpochBasedTrainer + from .training_args import TrainingArgs, build_dataset_from_file + from .hooks import Hook, Priority else: _import_structure = { @@ -32,7 +34,9 @@ else: 'SiameseUIETrainer' ], 'nlp_trainer': ['NlpEpochBasedTrainer', 'VecoTrainer'], - 'trainer': ['EpochBasedTrainer'] + 'trainer': ['EpochBasedTrainer'], + 'training_args': ['TrainingArgs', 'build_dataset_from_file'], + 'hooks': ['Hook'] } import sys diff --git a/modelscope/trainers/cli_argument_parser.py b/modelscope/trainers/cli_argument_parser.py new file mode 100644 index 00000000..f183b9ea --- /dev/null +++ b/modelscope/trainers/cli_argument_parser.py @@ -0,0 +1,151 @@ +from argparse import Action, ArgumentDefaultsHelpFormatter, ArgumentParser +from dataclasses import fields +from typing import List + + +class CliArgumentParser(ArgumentParser): + """ Argument Parser to define and parse command-line args for training. + + Args: + training_args: dict or list of dict which defines different + paramters for training. + """ + + def __init__(self, training_args=None, **kwargs): + if 'formatter_class' not in kwargs: + kwargs['formatter_class'] = ArgumentDefaultsHelpFormatter + super().__init__(**kwargs) + self.training_args = training_args + self.define_args() + + def get_manual_args(self, args): + return [arg[2:] for arg in args if arg.startswith('--')] + + def _parse_known_args(self, args: List = None, namespace=None): + self.model_id = namespace.model if namespace is not None else None + if '--model' in args: + self.model_id = args[args.index('--model') + 1] + self.manual_args = self.get_manual_args(args) + return super()._parse_known_args(args, namespace) + + def print_help(self, file=None): + return super().print_help(file) + + def define_args(self): + if self.training_args is not None: + for f in fields(self.training_args): + arg_name = f.name + arg_attr = getattr(self.training_args, f.name) + name = f'--{arg_name}' + kwargs = dict(type=f.type, help=f.metadata['help']) + kwargs['default'] = arg_attr + + if 'choices' in f.metadata: + kwargs['choices'] = f.metadata['choices'] + + kwargs['action'] = SingleAction + self.add_argument(name, **kwargs) + + +class DictAction(Action): + """ + argparse action to split an argument into KEY=VALUE form + on the first = and append to a dictionary. List options can + be passed as comma separated values, i.e 'KEY=V1,V2,V3', or with explicit + brackets, i.e. 'KEY=[V1,V2,V3]'. It also support nested brackets to build + list/tuple values. e.g. 'KEY=[(V1,V2),(V3,V4)]' + """ + + @staticmethod + def parse_int_float_bool_str(val): + try: + return int(val) + except ValueError: + pass + try: + return float(val) + except ValueError: + pass + if val.lower() in ['true', 'false']: + return val.lower() == 'true' + if val == 'None': + return None + return val + + @staticmethod + def parse_iterable(val): + """Parse iterable values in the string. + All elements inside '()' or '[]' are treated as iterable values. + Args: + val (str): Value string. + Returns: + list | tuple: The expanded list or tuple from the string. + Examples: + >>> DictAction._parse_iterable('1,2,3') + [1, 2, 3] + >>> DictAction._parse_iterable('[a, b, c]') + ['a', 'b', 'c'] + >>> DictAction._parse_iterable('[(1, 2, 3), [a, b], c]') + [(1, 2, 3), ['a', 'b'], 'c'] + """ + + def find_next_comma(string): + """Find the position of next comma in the string. + If no ',' is found in the string, return the string length. All + chars inside '()' and '[]' are treated as one element and thus ',' + inside these brackets are ignored. + """ + assert (string.count('(') == string.count(')')) and ( + string.count('[') + == string.count(']')), f'Imbalanced brackets exist in {string}' + end = len(string) + for idx, char in enumerate(string): + pre = string[:idx] + # The string before this ',' is balanced + if ((char == ',') and (pre.count('(') == pre.count(')')) + and (pre.count('[') == pre.count(']'))): + end = idx + break + return end + + # Strip ' and " characters and replace whitespace. + val = val.strip('\'\"').replace(' ', '') + is_tuple = False + if val.startswith('(') and val.endswith(')'): + is_tuple = True + val = val[1:-1] + elif val.startswith('[') and val.endswith(']'): + val = val[1:-1] + elif ',' not in val: + # val is a single value + return DictAction.parse_int_float_bool_str(val) + + values = [] + while len(val) > 0: + comma_idx = find_next_comma(val) + element = DictAction.parse_iterable(val[:comma_idx]) + values.append(element) + val = val[comma_idx + 1:] + if is_tuple: + values = tuple(values) + return values + + def __call__(self, parser, namespace, values, option_string): + options = {} + for kv in values: + key, val = kv.split('=', maxsplit=1) + options[key] = self.parse_iterable(val) + setattr(namespace, self.dest, options) + + +class SingleAction(DictAction): + """ Argparse action to convert value to tuple or list or nested structure of + list and tuple, i.e 'V1,V2,V3', or with explicit brackets, i.e. '[V1,V2,V3]'. + It also support nested brackets to build list/tuple values. e.g. '[(V1,V2),(V3,V4)]' + """ + + def __call__(self, parser, namespace, value, option_string): + if isinstance(value, str): + setattr(namespace, self.dest, self.parse_iterable(value)) + else: + setattr(namespace, self.dest, value) diff --git a/modelscope/trainers/default_config.py b/modelscope/trainers/default_config.py index 51a0df40..bb272695 100644 --- a/modelscope/trainers/default_config.py +++ b/modelscope/trainers/default_config.py @@ -4,38 +4,6 @@ from typing import Dict, List, Optional, Tuple from modelscope.utils.config import Config -DEFAULT_CONFIG = Config({ - 'framework': 'pytorch', - 'train': { - 'work_dir': '/tmp', - 'max_epochs': 10, - 'dataloader': { - 'batch_size_per_gpu': 16, - 'workers_per_gpu': 0 - }, - 'optimizer': { - 'type': 'SGD', - 'lr': 1e-3 - }, - 'lr_scheduler': { - 'type': 'StepLR', - 'step_size': 2 - }, - 'checkpoint': { - 'period': { - 'interval': 1 - } - } - }, - 'evaluation': { - 'dataloader': { - 'batch_size_per_gpu': 16, - 'workers_per_gpu': 0, - 'shuffle': False - }, - } -}) - DEFAULT_HOOKS_CONFIG = { 'train.hooks': [{ 'type': 'CheckpointHook', @@ -68,7 +36,7 @@ def merge_cfg(cfg: Config): def merge_hooks(cfg: Config) -> List[Dict]: - hooks = cfg.train.hooks.copy() + hooks = getattr(cfg.train, 'hooks', []).copy() for hook_type, key_chain in _HOOK_KEY_CHAIN_MAP.items(): hook = _key_chain_to_hook(cfg, key_chain, hook_type) if hook is not None: @@ -107,7 +75,8 @@ def _check_basic_hook(cfg: Config, key_chain: str, hook_type: str) -> bool: if cfg.safe_get(key_chain) is None: return False hooks = list( - filter(lambda hook: hook['type'] == hook_type, cfg.train.hooks)) + filter(lambda hook: hook['type'] == hook_type, + getattr(cfg.train, 'hooks', []))) assert len(hooks) == 0, f'The key_chain {key_chain} and the traditional hook ' \ f'cannot exist at the same time, ' \ f'please delete {hook_type} in the configuration file.' diff --git a/modelscope/trainers/hooks/__init__.py b/modelscope/trainers/hooks/__init__.py index 51677f25..5485f454 100644 --- a/modelscope/trainers/hooks/__init__.py +++ b/modelscope/trainers/hooks/__init__.py @@ -5,7 +5,6 @@ from modelscope.utils.import_utils import LazyImportModule if TYPE_CHECKING: from .builder import HOOKS, build_hook - from .checkpoint_hook import BestCkptSaverHook, CheckpointHook, LoadCheckpointHook from .early_stop_hook import EarlyStopHook from .compression import SparsityHook from .evaluation_hook import EvaluationHook @@ -16,6 +15,8 @@ if TYPE_CHECKING: from .optimizer import (ApexAMPOptimizerHook, NoneOptimizerHook, OptimizerHook, TorchAMPOptimizerHook) from .priority import Priority, get_priority + from .checkpoint import CheckpointHook, LoadCheckpointHook, BestCkptSaverHook + from .distributed import DDPHook, DeepspeedHook, MegatronHook else: _import_structure = { @@ -32,6 +33,9 @@ else: 'ApexAMPOptimizerHook', 'NoneOptimizerHook', 'OptimizerHook', 'TorchAMPOptimizerHook' ], + 'checkpoint': + ['CheckpointHook', 'LoadCheckpointHook', 'BestCkptSaverHook'], + 'distributed': ['DDPHook', 'DeepspeedHook', 'MegatronHook'], 'priority': ['Priority', 'get'] } diff --git a/modelscope/trainers/hooks/checkpoint/__init__.py b/modelscope/trainers/hooks/checkpoint/__init__.py new file mode 100644 index 00000000..e2abb272 --- /dev/null +++ b/modelscope/trainers/hooks/checkpoint/__init__.py @@ -0,0 +1,2 @@ +from .checkpoint_hook import BestCkptSaverHook, CheckpointHook +from .load_checkpoint_hook import LoadCheckpointHook diff --git a/modelscope/trainers/hooks/checkpoint/checkpoint_hook.py b/modelscope/trainers/hooks/checkpoint/checkpoint_hook.py new file mode 100644 index 00000000..e531a325 --- /dev/null +++ b/modelscope/trainers/hooks/checkpoint/checkpoint_hook.py @@ -0,0 +1,435 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +import random +import time +from typing import Optional + +import numpy as np +import torch + +from modelscope.hub.check_model import check_model_is_id +from modelscope.hub.push_to_hub import push_to_hub_async +from modelscope.metainfo import Hooks +from modelscope.trainers.hooks.builder import HOOKS +from modelscope.trainers.hooks.checkpoint.checkpoint_processor import \ + CheckpointProcessor +from modelscope.trainers.hooks.hook import Hook +from modelscope.trainers.hooks.priority import Priority +from modelscope.utils.constant import (DEFAULT_REPOSITORY_REVISION, LogKeys, + ModelFile) +from modelscope.utils.logger import get_logger +from modelscope.utils.torch_utils import is_master + + +class CheckpointStrategy: + by_epoch = 'by_epoch' + by_step = 'by_step' + no = 'no' + + +@HOOKS.register_module(module_name=Hooks.CheckpointHook) +class CheckpointHook(Hook): + """Save checkpoints periodically. + + Args: + save_strategy(str): The strategy to save checkpoint, can be `by_epoch`, `by_step` or `no` + interval (int): The frequency to save model. If `by_epoch=True`, + it means the number of epochs, else means the number of iterations + save_dir (str): The directory to save checkpoints. If is None, use `trainer.work_dir` + output_dir (str): The absolute path to save the output files for inference. If it's not specified, + the default dir is `{sub_dir}/output`. + save_last (bool): Whether to save the last checkpoint. Default: True. + max_checkpoint_num (int): The max number of checkpoint files, default None which means never delete anything. + If the number exceeding the limit, earlier checkpoints will be deleted first. + push_to_hub (bool): Whether push the checkpoint to modelhub. + hub_repo_id (str): The hub repo id. + hub_token (str): The token of the modelhub. You can also set the environment variable `MODELSCOPE_API_TOKEN`. + private_hub (bool): Whether push to a private hub, default True. + hub_revision (str): Which branch to push the model to, default is `master` + kwargs: + by_epoch (bool): Same with `save_strategy`, but has a higher priority, legacy argument. + output_sub_dir (str): The folder under the `save_dir` to save the output checkpoint for inference. + This argument is kept to fit the existing configs. + """ + + PRIORITY = Priority.LOW + + EVAL_RESULT_FILE = 'eval_result.txt' + + def __init__(self, + save_strategy: Optional[str] = CheckpointStrategy.by_epoch, + interval: Optional[int] = 0, + save_dir: Optional[str] = None, + output_dir: Optional[str] = None, + save_last: Optional[bool] = True, + max_checkpoint_num: Optional[int] = None, + push_to_hub: Optional[bool] = False, + hub_repo_id: Optional[str] = None, + hub_token: Optional[str] = None, + private_hub: Optional[bool] = True, + hub_revision: Optional[str] = DEFAULT_REPOSITORY_REVISION, + **kwargs): + self.interval = interval + self.save_dir = save_dir + if 'by_epoch' in kwargs: + self.save_strategy = CheckpointStrategy.by_epoch if kwargs[ + 'by_epoch'] else CheckpointStrategy.by_step + else: + self.save_strategy = save_strategy + if 'output_sub_dir' in kwargs: + self.output_sub_dir = kwargs['output_sub_dir'] + self.output_dir = None + else: + self.output_sub_dir = None + self.output_dir = output_dir + self.save_last = save_last + self.rng_state = None + self.push_to_hub = push_to_hub + self.hub_repo_id = hub_repo_id + self.hub_token = hub_token + self.private_hub = private_hub + self.hub_revision = hub_revision + self.tag = -1 + self.is_model_id = None + self.push_to_hub_future = None + self.max_checkpoint_num = None + if max_checkpoint_num is not None: + self.max_checkpoint_num = max(int(max_checkpoint_num), 1) + self.history_checkpoints = [] + self.processor = CheckpointProcessor() + + def set_processor(self, processor): + """ + The checkpoint hook accepts a processor to finish the actual saving/deleting action. + """ + self.processor = processor + + def before_run(self, trainer): + self.tag = -1 + if not self.save_dir: + self.save_dir = trainer.work_dir + if not self.output_dir: + if self.output_sub_dir: + self.output_dir = os.path.join(self.save_dir, + self.output_sub_dir) + else: + self.output_dir = os.path.join(self.save_dir, + ModelFile.TRAIN_OUTPUT_DIR) + + if not os.path.exists(self.save_dir): + os.makedirs(self.save_dir, exist_ok=True) + + if not hasattr(trainer, 'logger'): + self.logger = get_logger() + else: + self.logger = trainer.logger + + if is_master(): + output_dir = self.output_dir + # only global master prepares the output folder + self.processor.prepare_output(trainer, output_dir) + self.logger.info(f'Checkpoints will be saved to {self.save_dir}') + + def generate_prefix(self, trainer, save_strategy): + if save_strategy == CheckpointStrategy.by_epoch: + return f'{LogKeys.EPOCH}_{trainer.epoch + 1}' + else: + return f'{LogKeys.ITER}_{trainer.iter + 1}' + + def _do_save(self, trainer, save_strategy): + # prefix like 'epoch-1' or 'iter-1' + prefix = self.generate_prefix(trainer, save_strategy) + if self.processor.should_save_on_rank(trainer): + if is_master(): + if save_strategy == CheckpointStrategy.by_epoch: + self.logger.info( + f'Saving checkpoint at {trainer.epoch + 1} epoch') + else: + self.logger.info( + f'Saving checkpoint at {trainer.iter + 1} iter') + self._save_checkpoint(trainer, prefix) + if is_master() and self.push_to_hub: + if self.push_to_hub_future is not None and not self.push_to_hub_future.done( + ): + self.logger.error( + f'Another uploading is running, ' + f'this uploading with message {prefix} will be canceled.') + return + self.push_to_hub_future = self._push_to_hub(trainer, prefix) + + def after_train_epoch(self, trainer): + if self.save_strategy != CheckpointStrategy.by_epoch: + return + + if self._should_save(trainer): + self._do_save(trainer, CheckpointStrategy.by_epoch) + + def after_train_iter(self, trainer): + if self.save_strategy != CheckpointStrategy.by_step: + return + + if self._should_save(trainer): + self._do_save(trainer, CheckpointStrategy.by_step) + + def after_run(self, trainer): + if self.push_to_hub_future is not None and not self.push_to_hub_future.done( + ): + self.logger.info('Train finished. Uploading models, waiting...') + while not self.push_to_hub_future.done(): + time.sleep(1) + self.logger.info('Uploading models done.') + + def _push_to_hub(self, trainer, prefix): + if self.is_model_id is None: + self.is_model_id = check_model_is_id(trainer.input_model_id, + self.hub_token) + self.tag += 1 + return push_to_hub_async( + self.hub_repo_id, + self.output_dir, + token=self.hub_token, + private=self.private_hub, + commit_message=prefix, + tag=f'v1.{self.tag}', + revision=self.hub_revision, + source_repo=trainer.input_model_id if self.is_model_id else '') + + def save_evaluate_results(self, trainer): + with open(os.path.join(self.output_dir, self.EVAL_RESULT_FILE), + 'w') as f: + f.write(str(trainer.metric_values)) + + def _save_checkpoint(self, trainer, prefix): + """Save checkpoint files and remove obsolete ones + """ + checkpoint_path_prefix = os.path.join(self.save_dir, prefix) + meta = self._create_training_state(trainer) + self.processor.save_checkpoints(trainer, checkpoint_path_prefix, + self.output_dir, meta) + self.save_evaluate_results(trainer) + self.history_checkpoints.append(checkpoint_path_prefix) + self._remove_obsolete_checkpoints(trainer) + return prefix + + def _remove_obsolete_checkpoints(self, trainer): + if self.max_checkpoint_num is not None and \ + len(self.history_checkpoints) > self.max_checkpoint_num: + history_checkpoints = [ckpt for ckpt in self.history_checkpoints] + self.history_checkpoints.clear() + for i, checkpoint_path_prefix in enumerate(history_checkpoints): + if i < len(history_checkpoints) - self.max_checkpoint_num: + self.logger.info( + f'deleting checkpoint: {checkpoint_path_prefix}') + self.processor.remove_checkpoints( + trainer, checkpoint_path_prefix=checkpoint_path_prefix) + else: + self.history_checkpoints.append(checkpoint_path_prefix) + + def _should_save(self, trainer): + if self.save_strategy == CheckpointStrategy.by_epoch: + check_last = self.is_last_epoch + check_frequency = self.every_n_epochs + elif self.save_strategy == CheckpointStrategy.by_step: + check_last = self.is_last_iter + check_frequency = self.every_n_iters + else: + return False + + if check_frequency(trainer, + self.interval) or (self.save_last + and check_last(trainer)): + return True + return False + + def _create_training_state(self, trainer): + self.rng_state = { + 'random': random.getstate(), + 'numpy': np.random.get_state(), + 'cpu': torch.random.get_rng_state(), + 'cuda': torch.cuda.get_rng_state_all(), + } + + # keep epoch/iter/inner_iter/random_state + meta = { + 'epoch': trainer.epoch, + 'iter': trainer.iter + 1, + 'inner_iter': trainer.inner_iter + 1, + 'rng_state': self.rng_state, + } + + # keep hooks state + i = 0 + for hook in trainer.hooks: + if hasattr(hook, 'state_dict') and getattr(hook, '_should_save', + True): + meta[f'{hook.__class__}-{i}'] = hook.state_dict() + i += 1 + + return meta + + +@HOOKS.register_module(module_name=Hooks.BestCkptSaverHook) +class BestCkptSaverHook(CheckpointHook): + """ + Save best checkpoints hook. + + Args: + metric_key (str): Metric key to compare rule for best score. + save_best(bool): Save the best checkpoint, if set to False, this hook will have no effect. + rule (str): Comparison rule for best score. Support "max" and "min". If rule is "max", the checkpoint + at the maximum `metric_key` will be saved, If rule is "min", the checkpoint at the minimum `metric_key` + will be saved. + save_file_name: The manual specified saving file name. + restore_best (bool): Whether to restore the best checkpoint after training. + max_checkpoint_num (int): The max number of checkpoint files, default None which means never delete anything. + If the number exceeding the limit, checkpoints with worse metric will be deleted, which is judged by the + `rule` and `metric_key` arguments. + + The `BestCkptSaverHook` class accepts `output_sub_dir` and `output_dir` argument as its super class do. + If neither of them are passed, the default value is `{save_dir}/output_best`. + + This class will not accept the `interval` or `save_strategy` or `by_epoch` argument, because the saving interval + will follow the `EvaluationHook`. + """ + + PRIORITY = Priority.LOW + rule_map = {'max': lambda x, y: x > y, 'min': lambda x, y: x < y} + + def __init__(self, + metric_key: str, + save_best: Optional[bool] = True, + rule: Optional[str] = 'max', + save_file_name: Optional[str] = None, + restore_best: Optional[bool] = False, + max_checkpoint_num: Optional[int] = 1, + **kwargs): + assert rule in ['max', 'min'], 'Only support "max" or "min" rule now.' + output_kwargs = {} + if 'output_sub_dir' not in kwargs and 'output_dir' not in kwargs: + output_kwargs['output_sub_dir'] = ModelFile.TRAIN_BEST_OUTPUT_DIR + kwargs.pop('interval', None) + kwargs.pop('save_strategy', None) + super().__init__( + max_checkpoint_num=max_checkpoint_num, + **kwargs, + **output_kwargs, + ) + self.save_best = save_best + self.metric_key = metric_key + self.rule = rule + self._best_metric = None + self._best_ckpt_file = None + self.save_file_name = save_file_name + self.restore_best = restore_best + self.history_checkpoints = set() + + def after_train_epoch(self, trainer): + from modelscope.trainers.hooks import EvaluationHook + eval_hook = trainer.get_hook(EvaluationHook) + if len(eval_hook) == 0: + self.logger.error( + 'Trying to save the best checkpoint, but there is no evaluation, skipping.' + ) + + if eval_hook[0].last_eval_tag == ( + 'epoch', trainer.epoch) and self._should_save(trainer): + self._do_save(trainer, 'by_epoch') + + def after_train_iter(self, trainer): + from modelscope.trainers.hooks import EvaluationHook + eval_hook = trainer.get_hook(EvaluationHook) + if len(eval_hook) == 0: + self.logger.error( + 'Trying to save the best checkpoint, but there is no evaluation, skipping.' + ) + + if eval_hook[0].last_eval_tag == ( + 'iter', trainer.iter) and self._should_save(trainer): + self._do_save(trainer, 'by_step') + + def _should_save(self, trainer): + return self._is_best_metric(trainer.metric_values) and self.save_best + + def _is_best_metric(self, metric_values): + if metric_values is None: + return False + + if self.metric_key not in metric_values: + raise ValueError( + f'Not find metric_key: {self.metric_key} in {metric_values}') + + if self._best_metric is None: + self._best_metric = metric_values[self.metric_key] + return True + else: + compare_fn = self.rule_map[self.rule] + if compare_fn(metric_values[self.metric_key], self._best_metric): + self._best_metric = metric_values[self.metric_key] + return True + return False + + def generate_prefix(self, trainer, save_strategy): + if save_strategy == CheckpointStrategy.by_epoch: + return f'best_{LogKeys.EPOCH}{trainer.epoch + 1}_{self.metric_key}{self._best_metric}' + else: + return f'best_{LogKeys.ITER}{trainer.iter + 1}_{self.metric_key}{self._best_metric}' + + def _save_checkpoint(self, trainer, prefix): + checkpoint_path_prefix = self.save_file_name + if checkpoint_path_prefix is None: + checkpoint_path_prefix = os.path.join(self.save_dir, prefix) + else: + checkpoint_path_prefix = os.path.join(self.save_dir, + checkpoint_path_prefix) + + self._best_ckpt_file = checkpoint_path_prefix + meta = self._create_training_state(trainer) + self.processor.save_checkpoints(trainer, checkpoint_path_prefix, + self.output_dir, meta) + self.save_evaluate_results(trainer) + self.history_checkpoints.add(checkpoint_path_prefix) + self._remove_obsolete_checkpoints(trainer) + return prefix + + def _remove_obsolete_checkpoints(self, trainer): + + def extract_metric_from_filename(name1): + metric1 = float(name1.split(self.metric_key)[1]) + if self.rule == 'max': + return -metric1 + else: + return metric1 + + if self.max_checkpoint_num is not None and \ + len(self.history_checkpoints) > self.max_checkpoint_num: + history_checkpoints = sorted( + self.history_checkpoints, key=extract_metric_from_filename) + self.history_checkpoints.clear() + for i, checkpoint_path_prefix in enumerate(history_checkpoints): + if i < self.max_checkpoint_num: + self.history_checkpoints.add(checkpoint_path_prefix) + else: + self.logger.info( + f'deleting checkpoint: {checkpoint_path_prefix}') + self.processor.remove_checkpoints( + trainer, checkpoint_path_prefix=checkpoint_path_prefix) + + def state_dict(self): + return { + 'best_metric': self._best_metric, + } + + def load_state_dict(self, state_dict): + if state_dict is not None and len(state_dict) > 0: + self._best_metric = state_dict.get('best_metric') + else: + self.logger.warning( + 'The state_dict is not available, the best metric value will be affected.' + ) + + def after_run(self, trainer): + if self.restore_best: + # If restore_best is True, will call the LoadCheckpointHook to load the best checkpoint + # for later evaluation or prediction. + from modelscope.trainers.hooks.checkpoint.load_checkpoint_hook import LoadCheckpointHook + LoadCheckpointHook.load_checkpoint(self._best_ckpt_file, trainer) diff --git a/modelscope/trainers/hooks/checkpoint/checkpoint_processor.py b/modelscope/trainers/hooks/checkpoint/checkpoint_processor.py new file mode 100644 index 00000000..f28fc397 --- /dev/null +++ b/modelscope/trainers/hooks/checkpoint/checkpoint_processor.py @@ -0,0 +1,276 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +import re +import shutil + +from modelscope.metainfo import Pipelines +from modelscope.utils.checkpoint import (load_checkpoint, save_checkpoint, + save_configuration) +from modelscope.utils.constant import ModelFile +from modelscope.utils.logger import get_logger +from modelscope.utils.torch_utils import is_master + + +class CheckpointProcessor: + + TRAINER_STATE_SUFFIX = '_trainer_state.pth' + + MODEL_STATE_SUFFIX = '.pth' + + def prepare_output(self, trainer, output_dir): + """Prepares the output of target folder. + + This is a strategic function which can be registered by other hook's function. + + Args: + trainer: The trainer instance. + output_dir: The target folder used in inference. + """ + model = trainer.unwrap_module(trainer.model) + config = trainer.cfg + + # override pipeline by tasks name after finetune done, + # avoid case like fill mask pipeline with a text cls task + if config['task'] in [ + getattr(Pipelines, attr) for attr in dir(Pipelines) + if not attr.startswith('__') + ]: + # TODO a temp fix to avoid pipeline_name and task mismatch + config['pipeline'] = {'type': config['task']} + + self.copy_files_and_dump_config(trainer, output_dir, config, + self._bin_file(model)) + + @staticmethod + def copy_files_and_dump_config(trainer, output_dir, config, bin_file): + """Copy useful files to target output folder and dumps the target configuration.json. + """ + model = trainer.unwrap_module(trainer.model) + + class SaveConfig: + + def __init__(self, output_dir, config): + self.output_dir = output_dir + self.config = config + + def __call__(self, _output_dir, _config): + self.config = _config + + def save_config(self): + save_configuration(self.output_dir, self.config) + + for pop_key in [ + 'push_to_hub', 'hub_repo_id', 'hub_token', 'private_hub' + ]: + if config.safe_get('train.checkpoint.period.' + + pop_key) is not None: + config.safe_get('train.checkpoint.period').pop(pop_key) + if config.safe_get('train.checkpoint.best.' + pop_key) is not None: + config.safe_get('train.checkpoint.best').pop(pop_key) + + save_config_fn = SaveConfig(output_dir, config) + + if hasattr(model, 'save_pretrained'): + # Save pretrained of model, skip saving checkpoint + model.save_pretrained( + output_dir, + bin_file, + save_function=lambda *args, **kwargs: None, + config=save_config_fn.config, + save_config_function=save_config_fn) + + if trainer.train_preprocessor is not None: + trainer.train_preprocessor.save_pretrained( + output_dir, + save_config_fn.config, + save_config_function=save_config_fn) + if trainer.eval_preprocessor is not None: + trainer.eval_preprocessor.save_pretrained( + output_dir, + save_config_fn.config, + save_config_function=save_config_fn) + save_config_fn.save_config() + + @staticmethod + def _bin_file(model): + """Get bin file path. + """ + default_bin_file = ModelFile.TORCH_MODEL_BIN_FILE + if hasattr(model, + 'model_dir') and ModelFile.TORCH_MODEL_FILE in os.listdir( + model.model_dir): + default_bin_file = ModelFile.TORCH_MODEL_FILE + return default_bin_file + + def save_checkpoints(self, + trainer, + checkpoint_path_prefix, + output_dir, + meta=None): + """Save the state dict for trainer and model. + + This is a strategic function which can be registered by other hook's function. + + Args: + trainer(`EpochBasedTrainer`): The trainer instance. + checkpoint_path_prefix(`str`): The saving dir with a prefix. + like: /tmp/test/epoch_0 + output_dir(`str`): The output dir for inference. + meta: (`dict`): The meta info needed to be saved into files. + """ + model = trainer.unwrap_module(trainer.model) + _model_file, _train_state_file = self._get_state_file_name( + checkpoint_path_prefix) + + # Save pth file without model state_dict + self.save_trainer_state(trainer, model, _train_state_file, meta) + self.save_model_state(model, _model_file) + self.link(model, _model_file, output_dir) + + def remove_checkpoints(self, trainer, checkpoint_path_prefix): + """Remove obsolete checkpoint files. + + This is a strategic function which can be registered by other hook's function. + + Args: + trainer(`EpochBasedTrainer`): The trainer instance. + checkpoint_path_prefix(`str`): The saving dir with a prefix. + like: /tmp/test/epoch_0 + """ + _model_file, _train_state_file = self._get_state_file_name( + checkpoint_path_prefix) + if os.path.isfile(_train_state_file): + os.remove(_train_state_file) + + if os.path.isfile(_model_file): + os.remove(_model_file) + + def should_save_on_rank(self, trainer): + """Used in ddp or other distributed training scenario, returns whether do saving in current rank. + + This is a strategic function which can be registered by other hook's function. + + Args: + trainer(`EpochBasedTrainer`): The trainer instance. + """ + return is_master() + + def link(self, model, src_file, output_dir): + """Links the src bin file to the output folder. + + Args: + model: The model instance. + src_file: The src bin file path. + output_dir: The target folder used in inference. + """ + + bin_file = self._bin_file(model) + dest_file = os.path.join(output_dir, bin_file) + if os.path.isfile(dest_file): + os.unlink(dest_file) + + try: + os.link(src_file, dest_file) + except OSError as e: + get_logger().error( + f'Link {src_file} to {dest_file} error: {e}, ' + 'changing to copy the bin file, this may use more disk space.') + shutil.copyfile(src_file, dest_file) + + def save_trainer_state(self, trainer, model, train_state_file, meta): + """Save the trainer state, including optimizer/lr_scheduler's state dict, random states etc. + + Args: + trainer: The trainer instance. + model: The model instance. + train_state_file: The target file name for saving trainer states. + meta: Some extra meta info. + """ + save_checkpoint( + model, + train_state_file, + trainer.optimizer, + trainer.lr_scheduler, + meta=meta, + with_model=False) + + def save_model_state(self, model, model_file): + """Save the model state. + + Args: + model: The model instance. + model_file: The target file name for saving model states. + """ + save_checkpoint( + model, model_file, None, None, meta=None, with_meta=False) + + def load_checkpoints(self, checkpoint_path_prefix, trainer, load_all_state, + strict): + """Load checkpoint files of trainer state and model state. + + This is a strategic function which can be registered by other hook's function. + + Args: + checkpoint_path_prefix(str): The checkpoint dir with prefix or a model state file. + Example: '/tmp/test/epoch_0' or '/tmp/test/epoch_0.pth' + trainer(`EpochBasedTrainer`): The trainer instance. + load_all_state(`boolean`): Load all states (else load only module states). + strict(`boolean`): If strict, any unmatched keys will cause an error. + + Returns: + The meta info in json. + """ + _model_file, _train_state_file = self._get_state_file_name( + checkpoint_path_prefix) + meta = {} + if os.path.isfile(_train_state_file): + meta = self.load_trainer_state(trainer, _train_state_file, + load_all_state) + else: + print(f'No trainer state file {_train_state_file} found, skip.') + self.load_model_state(trainer, _model_file, strict) + return meta + + @staticmethod + def load_trainer_state(trainer, train_state_file, load_all_state): + """Load trainer state file. + """ + + optimizer = getattr(trainer, 'optimizer', + None) if load_all_state else None + lr_scheduler = getattr(trainer, 'lr_scheduler', + None) if load_all_state else None + return load_checkpoint(train_state_file, None, optimizer, lr_scheduler) + + def load_model_state(self, trainer, model_file, strict): + """Load model state file. + """ + return load_checkpoint(model_file, + trainer.unwrap_module(trainer.model), None, + None) + + @staticmethod + def _get_state_file_name(checkpoint_path_prefix): + """Get the default file name for state files. + + If the input is a checkpoint dir with prefix, this function will append suffix for both checkpoint files. + If the input is an absolute file name, this function will return it as the model file name, and append + suffix for the trainer file name. + + NOTE: a best checkpoint filename with float or int metric value inside + will not be judged as having a extension file name. like: '/tmp/test/epoch_0_accuracy0.85' + + Args: + checkpoint_path_prefix(`str`): The checkpoint dir with prefix or a model state file + with extension file name. like: '/tmp/test/epoch_0' + + Returns: + A tuple of model state file name and trainer state file name. + """ + base, ext = os.path.splitext(checkpoint_path_prefix) + if len(ext) == 0 or re.match(r'^\d+$', ext[1:]): + return checkpoint_path_prefix + CheckpointProcessor.MODEL_STATE_SUFFIX, \ + checkpoint_path_prefix + CheckpointProcessor.TRAINER_STATE_SUFFIX # noqa + else: + return checkpoint_path_prefix, base + CheckpointProcessor.TRAINER_STATE_SUFFIX.split( + '.')[0] + '.' + ext[1:] diff --git a/modelscope/trainers/hooks/checkpoint/load_checkpoint_hook.py b/modelscope/trainers/hooks/checkpoint/load_checkpoint_hook.py new file mode 100644 index 00000000..3ccb800f --- /dev/null +++ b/modelscope/trainers/hooks/checkpoint/load_checkpoint_hook.py @@ -0,0 +1,138 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import random +from typing import Optional + +import numpy as np +import torch +from packaging import version + +from modelscope.metainfo import Hooks +from modelscope.trainers.hooks.builder import HOOKS +from modelscope.trainers.hooks.checkpoint.checkpoint_processor import \ + CheckpointProcessor +from modelscope.trainers.hooks.hook import Hook +from modelscope.trainers.hooks.priority import Priority +from modelscope.utils.logger import get_logger + + +@HOOKS.register_module(module_name=Hooks.LoadCheckpointHook) +class LoadCheckpointHook(Hook): + """Load a checkpoint file at the beginning of training or evaluating. + + This hook does not need to be configured or saved in the config file. + User should use it by: + >>> trainer.train('some-checkpoint', load_all_state=True) + or + >>> trainer.evaluate('some-checkpoint') + instead. + + Args: + checkpoint_file (str): The checkpoint file to be loaded. + load_all_state (bool): Load all states(optimizer, epoch, lr_scheduler, random_state, etc.) when loading old + training state file or not. The model's state dict will only be loaded if False. + strict (bool): If strict, any unmatched keys will cause an error. + """ + + PRIORITY = Priority.HIGH + + _should_save = False + + # From 1.3.1 version we split one pth file to two files: trainer state pth file/model state pth file. + _TWO_PTH_FILE_VERSION = '1.3.1' + + def __init__( + self, + checkpoint_file: Optional[str] = None, + load_all_state: Optional[bool] = True, + strict: Optional[bool] = False, + ): + self.checkpoint_file = checkpoint_file + self.rng_state = None + self.need_load_rng_state = False + self.load_all_state = load_all_state + self.strict = strict + self.processor = CheckpointProcessor() + + def before_run(self, trainer): + if not hasattr(trainer, 'logger'): + self.logger = get_logger() + else: + self.logger = trainer.logger + + if self.checkpoint_file is not None: + meta = self.load_checkpoint(self.checkpoint_file, trainer, + self.load_all_state, self.strict) + self.rng_state = meta.get('rng_state') + self.need_load_rng_state = self.load_all_state + + def before_train_iter(self, trainer): + if self.need_load_rng_state: + if self.rng_state is not None: + random.setstate(self.rng_state['random']) + np.random.set_state(self.rng_state['numpy']) + torch.random.set_rng_state(self.rng_state['cpu']) + if torch.cuda.is_available(): + torch.cuda.random.set_rng_state_all(self.rng_state['cuda']) + self.need_load_rng_state = False + else: + self.logger.info( + 'Random state cannot be found in checkpoint file, ' + 'this may cause a random data order or model initialization.' + ) + + @staticmethod + def _restore_training_state(trainer, meta): + trainer._epoch = meta.get('epoch', trainer._epoch) + trainer._iter = meta.get('iter', trainer._iter) + trainer._inner_iter = meta.get('inner_iter', trainer._inner_iter) + + i = 0 + for hook in trainer.hooks: + if hasattr(hook, 'load_state_dict') and getattr( + hook, '_should_save', True): + key = f'{hook.__class__}-{i}' + if key in meta: + hook.load_state_dict(meta.get(key, {})) + else: + trainer.logger.warning( + f'The state_dict of hook {hook.__class__} at index {i} is not found in the checkpoint file.' + ) + i += 1 + + @classmethod + def load_checkpoint(cls, + filename, + trainer, + load_all_state=True, + strict=False): + """A static method to load checkpoint files. + + Args: + filename(str): An absolute model bin file(pth or bin) or a dir path with a file prefix(like epoch_1). + trainer(`EpochBasedTrainer`): The trainer instance. + load_all_state(`bool`): Load all states including the trainer states. + strict(`bool`): Load module state dict strictly. + + Returns: + A dict containing the train states saved by `_create_training_state` + """ + meta = cls().processor.load_checkpoints(filename, trainer, + load_all_state, strict) + if load_all_state: + cls._restore_training_state(trainer, meta) + + if meta is not None: + _version = meta.get('modelscope') + if _version is not None and version.parse( + _version) < version.parse( + LoadCheckpointHook._TWO_PTH_FILE_VERSION): + trainer.logger.warning( + 'The unique pth file is split into a model file and ' + f'a trainer file since version {LoadCheckpointHook._TWO_PTH_FILE_VERSION},' + 'consider re-training your model or ' + 'using a converting script to split the single pth file into two.' + ) + trainer.logger.info( + f'Checkpoint {filename} saving time: {meta.get("time")}, modelscope version: {_version}' + ) + return meta diff --git a/modelscope/trainers/hooks/checkpoint_hook.py b/modelscope/trainers/hooks/checkpoint_hook.py deleted file mode 100644 index 59832105..00000000 --- a/modelscope/trainers/hooks/checkpoint_hook.py +++ /dev/null @@ -1,749 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import os -import random -import re -import time - -import numpy as np -import torch -from packaging import version - -from modelscope.hub.check_model import check_model_is_id -from modelscope.hub.push_to_hub import push_to_hub_async -from modelscope.metainfo import Hooks, Pipelines -from modelscope.utils.checkpoint import (load_checkpoint, save_checkpoint, - save_configuration) -from modelscope.utils.constant import LogKeys, ModelFile -from modelscope.utils.logger import get_logger -from modelscope.utils.torch_utils import is_master -from .builder import HOOKS -from .hook import Hook -from .priority import Priority - - -@HOOKS.register_module(module_name=Hooks.CheckpointHook) -class CheckpointHook(Hook): - """Save checkpoints periodically. - - Args: - interval (int): The frequency to save model. If `by_epoch=True`, - it means the number of epochs, else means the number of iterations - by_epoch (bool): Saving checkpoints by epoch or by iteration. - save_optimizer (bool): Whether to save optimizer state dict. Default: True. - save_dir (str): The directory to save checkpoints. If is None, use `trainer.work_dir` - output_sub_dir (str): The sub folder under the `save_dir` to save the output checkpoint for inference. - Default 'output'. - save_last (bool): Whether to save the last checkpoint. Default: True. - max_checkpoint_num (int): The max number of checkpoint files, default None which means never delete anything. - If the number exceeding the limit, earlier checkpoints will be deleted first. - """ - - PRIORITY = Priority.LOW - - TRAINER_STATE_SUFFIX = '_trainer_state.pth' - - MODEL_STATE_SUFFIX = '.pth' - - def __init__(self, - interval=0, - by_epoch=True, - save_optimizer=True, - save_dir=None, - output_sub_dir=ModelFile.TRAIN_OUTPUT_DIR, - save_last=True, - max_checkpoint_num=None, - push_to_hub=False, - model_id_with_org=None, - hub_token=None, - private_hub=True, - **kwargs): - self.interval = interval - self.by_epoch = by_epoch - self.save_optimizer = save_optimizer - self.save_dir = save_dir - self.output_sub_dir = output_sub_dir - self.save_last = save_last - self.rng_state = None - self.max_checkpoint_num = None - self.push_to_hub = push_to_hub - self.model_id_with_org = model_id_with_org - self.hub_token = hub_token - self.private_hub = private_hub - self.is_model_id = None - self.push_to_hub_future = None - if max_checkpoint_num is not None: - self.max_checkpoint_num = max(int(max_checkpoint_num), 1) - self.history_checkpoints = [] - - def before_run(self, trainer): - if not self.save_dir: - self.save_dir = trainer.work_dir - - if not os.path.exists(self.save_dir): - os.makedirs(self.save_dir, exist_ok=True) - - if not hasattr(trainer, 'logger'): - self.logger = get_logger() - else: - self.logger = trainer.logger - - if is_master(): - output_dir = os.path.join(self.save_dir, self.output_sub_dir) - # only global master prepares the output folder - self.prepare_output(trainer, output_dir) - self.logger.info(f'Checkpoints will be saved to {self.save_dir}') - - def generate_prefix(self, trainer): - if self.by_epoch: - return f'{LogKeys.EPOCH}_{trainer.epoch + 1}' - else: - return f'{LogKeys.ITER}_{trainer.iter + 1}' - - def after_train_epoch(self, trainer): - if not self.by_epoch: - return - - if self._should_save(trainer): - # prefix like 'epoch-1' or 'iter-1' - prefix = self.generate_prefix(trainer) - if self.should_save_on_rank(trainer): - if is_master(): - self.logger.info( - f'Saving checkpoint at {trainer.epoch + 1} epoch') - self._save_checkpoint(trainer, prefix) - if is_master() and self.push_to_hub: - if self.push_to_hub_future is not None and not self.push_to_hub_future.done( - ): - self.logger.error( - f'Another uploading is running, ' - f'this uploading with message {prefix} will be canceled.' - ) - return - self.push_to_hub_future = self._push_to_hub(trainer, prefix) - - def after_train_iter(self, trainer): - if self.by_epoch: - return - - if self._should_save(trainer): - # prefix like 'epoch-1' or 'iter-1' - prefix = self.generate_prefix(trainer) - if self.should_save_on_rank(trainer): - if is_master(): - self.logger.info( - f'Saving checkpoint at {trainer.iter + 1} iter') - self._save_checkpoint(trainer, prefix) - if is_master() and self.push_to_hub: - if self.push_to_hub_future is not None and not self.push_to_hub_future.done( - ): - self.logger.error( - f'Another uploading is running, ' - f'this uploading with message {prefix} will be canceled.' - ) - return - self.push_to_hub_future = self._push_to_hub(trainer, prefix) - - def after_run(self, trainer): - if self.push_to_hub_future is not None and not self.push_to_hub_future.done( - ): - self.logger.info('Train finished. Uploading models, waiting...') - while not self.push_to_hub_future.done(): - time.sleep(1) - self.logger.info('Uploading models done.') - - def _push_to_hub(self, trainer, prefix): - if self.is_model_id is None: - self.is_model_id = check_model_is_id(trainer.input_model_id, - self.hub_token) - - return push_to_hub_async( - self.model_id_with_org, - os.path.join(self.save_dir, self.output_sub_dir), - token=self.hub_token, - private=self.private_hub, - commit_message=prefix, - source_repo=trainer.input_model_id if self.is_model_id else '') - - def _save_checkpoint(self, trainer, prefix): - """Save checkpoint files and remove obsolete ones - """ - checkpoint_path_prefix = os.path.join(self.save_dir, prefix) - meta = self._create_training_state(trainer) - self.save_checkpoints(trainer, checkpoint_path_prefix, - self.output_sub_dir, meta) - self.history_checkpoints.append(checkpoint_path_prefix) - self._remove_obsolete_checkpoints(trainer) - return prefix - - def _remove_obsolete_checkpoints(self, trainer): - if self.max_checkpoint_num is not None and \ - len(self.history_checkpoints) > self.max_checkpoint_num: - history_checkpoints = [ckpt for ckpt in self.history_checkpoints] - self.history_checkpoints.clear() - for i, checkpoint_path_prefix in enumerate(history_checkpoints): - if i < len(history_checkpoints) - self.max_checkpoint_num: - self.logger.info( - f'deleting checkpoint: {checkpoint_path_prefix}') - self.remove_checkpoints( - trainer, checkpoint_path_prefix=checkpoint_path_prefix) - else: - self.history_checkpoints.append(checkpoint_path_prefix) - - def _should_save(self, trainer): - if self.by_epoch: - check_last = self.is_last_epoch - check_frequency = self.every_n_epochs - else: - check_last = self.is_last_iter - check_frequency = self.every_n_iters - - if check_frequency(trainer, - self.interval) or (self.save_last - and check_last(trainer)): - return True - return False - - def _create_training_state(self, trainer): - self.rng_state = { - 'random': random.getstate(), - 'numpy': np.random.get_state(), - 'cpu': torch.random.get_rng_state(), - 'cuda': torch.cuda.get_rng_state_all(), - } - - # keep epoch/iter/inner_iter/random_state - meta = { - 'epoch': trainer.epoch, - 'iter': trainer.iter + 1, - 'inner_iter': trainer.inner_iter + 1, - 'rng_state': self.rng_state, - } - - # keep hooks state - i = 0 - for hook in trainer.hooks: - if hasattr(hook, 'state_dict') and getattr(hook, '_should_save', - True): - meta[f'{hook.__class__}-{i}'] = hook.state_dict() - i += 1 - - return meta - - @staticmethod - def copy_files_and_dump_config(trainer, output_dir, config, bin_file): - """Copy useful files to target output folder and dumps the target configuration.json. - """ - model = trainer.unwrap_module(trainer.model) - - class SaveConfig: - - def __init__(self, output_dir, config): - self.output_dir = output_dir - self.config = config - - def __call__(self, _output_dir, _config): - self.config = _config - - def save_config(self): - save_configuration(self.output_dir, self.config) - - for pop_key in [ - 'push_to_hub', 'model_id_with_org', 'hub_token', 'private_hub' - ]: - if config.safe_get('train.checkpoint.period.' - + pop_key) is not None: - config.safe_get('train.checkpoint.period').pop(pop_key) - if config.safe_get('train.checkpoint.best.' + pop_key) is not None: - config.safe_get('train.checkpoint.best').pop(pop_key) - - save_config_fn = SaveConfig(output_dir, config) - - if hasattr(model, 'save_pretrained'): - # Save pretrained of model, skip saving checkpoint - model.save_pretrained( - output_dir, - bin_file, - save_function=lambda *args, **kwargs: None, - config=save_config_fn.config, - save_config_function=save_config_fn) - - if trainer.train_preprocessor is not None: - trainer.train_preprocessor.save_pretrained( - output_dir, - save_config_fn.config, - save_config_function=save_config_fn) - if trainer.eval_preprocessor is not None: - trainer.eval_preprocessor.save_pretrained( - output_dir, - save_config_fn.config, - save_config_function=save_config_fn) - save_config_fn.save_config() - - @staticmethod - def _bin_file(model): - """Get bin file path. - """ - default_bin_file = ModelFile.TORCH_MODEL_BIN_FILE - if hasattr(model, - 'model_dir') and ModelFile.TORCH_MODEL_FILE in os.listdir( - model.model_dir): - default_bin_file = ModelFile.TORCH_MODEL_FILE - return default_bin_file - - @Hook.overload_func(name='CheckpointHook.prepare_output') - def prepare_output(self, trainer, output_dir): - """Prepares the output of target folder. - - This is a strategic function which can be registered by other hook's function. - - Args: - trainer: The trainer instance. - output_dir: The target folder used in inference. - """ - model = trainer.unwrap_module(trainer.model) - config = trainer.cfg - - # override pipeline by tasks name after finetune done, - # avoid case like fill mask pipeline with a text cls task - if config['task'] in [ - getattr(Pipelines, attr) for attr in dir(Pipelines) - if not attr.startswith('__') - ]: - # TODO a temp fix to avoid pipeline_name and task mismatch - config['pipeline'] = {'type': config['task']} - - self.copy_files_and_dump_config(trainer, output_dir, config, - self._bin_file(model)) - - def link(self, model, src_file, output_dir): - """Links the src bin file to the output folder. - - Args: - model: The model instance. - src_file: The src bin file path. - output_dir: The target folder used in inference. - """ - - bin_file = self._bin_file(model) - dest_file = os.path.join(output_dir, bin_file) - if os.path.isfile(dest_file): - os.unlink(dest_file) - - os.link(src_file, dest_file) - - def save_trainer_state(self, trainer, model, train_state_file, meta): - """Save the trainer state, including optimizer/lr_scheduler's state dict, random states etc. - - Args: - trainer: The trainer instance. - model: The model instance. - train_state_file: The target file name for saving trainer states. - meta: Some extra meta info. - """ - save_checkpoint( - model, - train_state_file, - trainer.optimizer, - trainer.lr_scheduler, - meta=meta, - with_model=False) - - def save_model_state(self, model, model_file): - """Save the model state. - - Args: - model: The model instance. - model_file: The target file name for saving model states. - """ - save_checkpoint( - model, model_file, None, None, meta=None, with_meta=False) - - @Hook.overload_func(name='CheckpointHook.save_checkpoints') - def save_checkpoints(self, - trainer, - checkpoint_path_prefix, - output_sub_dir, - meta=None): - """Save the state dict for trainer and model. - - This is a strategic function which can be registered by other hook's function. - - Args: - trainer(`EpochBasedTrainer`): The trainer instance. - checkpoint_path_prefix(`str`): The saving dir with a prefix. - like: /tmp/test/epoch_0 - output_sub_dir(`str`): The sub-dir in the saving dir used in inference. - meta: (`dict`): The meta info needed to be saved into files. - """ - model = trainer.unwrap_module(trainer.model) - _model_file, _train_state_file = _get_state_file_name( - checkpoint_path_prefix) - - # Save pth file without model state_dict - self.save_trainer_state(trainer, model, _train_state_file, meta) - self.save_model_state(model, _model_file) - output_dir = os.path.join(self.save_dir, output_sub_dir) - self.link(model, _model_file, output_dir) - - @Hook.overload_func(name='CheckpointHook.remove_checkpoints') - def remove_checkpoints(self, trainer, checkpoint_path_prefix): - """Remove obsolete checkpoint files. - - This is a strategic function which can be registered by other hook's function. - - Args: - trainer(`EpochBasedTrainer`): The trainer instance. - checkpoint_path_prefix(`str`): The saving dir with a prefix. - like: /tmp/test/epoch_0 - """ - _model_file, _train_state_file = _get_state_file_name( - checkpoint_path_prefix) - if os.path.isfile(_train_state_file): - os.remove(_train_state_file) - - if os.path.isfile(_model_file): - os.remove(_model_file) - - @Hook.overload_func(name='CheckpointHook.should_save_on_rank') - def should_save_on_rank(self, trainer): - """Used in ddp or other distributed training scenario, returns whether do saving in current rank. - - This is a strategic function which can be registered by other hook's function. - - Args: - trainer(`EpochBasedTrainer`): The trainer instance. - """ - return is_master() - - -@HOOKS.register_module(module_name=Hooks.BestCkptSaverHook) -class BestCkptSaverHook(CheckpointHook): - """ - Save best checkpoints hook. - - Args: - metric_key (str): Metric key to compare rule for best score. - rule (str): Comparison rule for best score. Support "max" and "min". If rule is "max", the checkpoint - at the maximum `metric_key` will be saved, If rule is "min", the checkpoint at the minimum `metric_key` - will be saved. - by_epoch (bool): Save best checkpoints by epoch or by iteration. - save_optimizer (bool): Whether to save optimizer state dict. Default: True. - save_dir (str): Output directory to save best checkpoint. - output_sub_dir (str): The sub folder under the `save_dir` to save the output checkpoint for inference. - Default 'output_best'. - restore_best (bool): Whether to restore the best checkpoint after training. - max_checkpoint_num (int): The max number of checkpoint files, default None which means never delete anything. - If the number exceeding the limit, checkpoints with worse metric will be deleted, which is judged by the - `rule` and `metric_key` arguments. - """ - - PRIORITY = Priority.LOW - rule_map = {'max': lambda x, y: x > y, 'min': lambda x, y: x < y} - - def __init__(self, - metric_key, - rule='max', - by_epoch=True, - save_optimizer=True, - save_dir=None, - output_sub_dir=ModelFile.TRAIN_BEST_OUTPUT_DIR, - save_file_name=None, - restore_best=False, - max_checkpoint_num=1, - interval=0, - **kwargs): - assert rule in ['max', 'min'], 'Only support "max" or "min" rule now.' - super().__init__( - interval=interval, - by_epoch=by_epoch, - save_optimizer=save_optimizer, - save_dir=save_dir, - output_sub_dir=output_sub_dir, - max_checkpoint_num=max_checkpoint_num, - **kwargs, - ) - self.metric_key = metric_key - self.rule = rule - self._best_metric = None - self._best_ckpt_file = None - self.save_file_name = save_file_name - self.restore_best = restore_best - self.history_checkpoints = set() - - def _should_save(self, trainer): - return self._is_best_metric(trainer.metric_values) - - def _is_best_metric(self, metric_values): - if metric_values is None: - return False - - if self.metric_key not in metric_values: - raise ValueError( - f'Not find metric_key: {self.metric_key} in {metric_values}') - - if self._best_metric is None: - self._best_metric = metric_values[self.metric_key] - return True - else: - compare_fn = self.rule_map[self.rule] - if compare_fn(metric_values[self.metric_key], self._best_metric): - self._best_metric = metric_values[self.metric_key] - return True - return False - - def generate_prefix(self, trainer): - if self.by_epoch: - return f'best_{LogKeys.EPOCH}{trainer.epoch + 1}_{self.metric_key}{self._best_metric}' - else: - return f'best_{LogKeys.ITER}{trainer.iter + 1}_{self.metric_key}{self._best_metric}' - - def _save_checkpoint(self, trainer, prefix): - checkpoint_path_prefix = self.save_file_name - if checkpoint_path_prefix is None: - checkpoint_path_prefix = os.path.join(self.save_dir, prefix) - else: - checkpoint_path_prefix = os.path.join(self.save_dir, - checkpoint_path_prefix) - - self._best_ckpt_file = checkpoint_path_prefix - meta = self._create_training_state(trainer) - self.save_checkpoints(trainer, checkpoint_path_prefix, - self.output_sub_dir, meta) - self.history_checkpoints.add(checkpoint_path_prefix) - self._remove_obsolete_checkpoints(trainer) - return prefix - - def _remove_obsolete_checkpoints(self, trainer): - - def extract_metric_from_filename(name1): - metric1 = float(name1.split(self.metric_key)[1]) - if self.rule == 'max': - return -metric1 - else: - return metric1 - - if self.max_checkpoint_num is not None and \ - len(self.history_checkpoints) > self.max_checkpoint_num: - history_checkpoints = sorted( - self.history_checkpoints, key=extract_metric_from_filename) - self.history_checkpoints.clear() - for i, checkpoint_path_prefix in enumerate(history_checkpoints): - if i < self.max_checkpoint_num: - self.history_checkpoints.add(checkpoint_path_prefix) - else: - self.logger.info( - f'deleting checkpoint: {checkpoint_path_prefix}') - self.remove_checkpoints( - trainer, checkpoint_path_prefix=checkpoint_path_prefix) - - def state_dict(self): - return { - 'best_metric': self._best_metric, - } - - def load_state_dict(self, state_dict): - if state_dict is not None and len(state_dict) > 0: - self._best_metric = state_dict.get('best_metric') - else: - self.logger.warning( - 'The state_dict is not available, the best metric value will be affected.' - ) - - def after_run(self, trainer): - if self.restore_best: - # If restore_best is True, will call the LoadCheckpointHook to load the best checkpoint - # for later evaluation or prediction. - LoadCheckpointHook.load_checkpoint(self._best_ckpt_file, trainer) - - -@HOOKS.register_module(module_name=Hooks.LoadCheckpointHook) -class LoadCheckpointHook(Hook): - """Load a checkpoint file at the beginning of training or evaluating. - - This hook does not need to be configured or saved in the config file. - User should use it by: - >>> trainer.train('some-checkpoint', load_all_state=True) - or - >>> trainer.evaluate('some-checkpoint') - instead. - - Args: - checkpoint_file (str): The checkpoint file to be loaded. - load_all_state (bool): Load all states(optimizer, epoch, lr_scheduler, random_state, etc.) when loading old - training state file or not. The model's state dict will only be loaded if False. - strict (bool): If strict, any unmatched keys will cause an error. - """ - - PRIORITY = Priority.HIGH - - _should_save = False - - _TWO_PTH_FILE_VERSION = '1.3.1' - - def __init__( - self, - checkpoint_file=None, - load_all_state=True, - strict=False, - ): - self.checkpoint_file = checkpoint_file - self.rng_state = None - self.need_load_rng_state = False - self.load_all_state = load_all_state - self.strict = strict - - def before_run(self, trainer): - if not hasattr(trainer, 'logger'): - self.logger = get_logger() - else: - self.logger = trainer.logger - - if self.checkpoint_file is not None: - meta = self.load_checkpoint(self.checkpoint_file, trainer, - self.load_all_state, self.strict) - self.rng_state = meta.get('rng_state') - self.need_load_rng_state = self.load_all_state - - def before_train_iter(self, trainer): - if self.need_load_rng_state: - if self.rng_state is not None: - random.setstate(self.rng_state['random']) - np.random.set_state(self.rng_state['numpy']) - torch.random.set_rng_state(self.rng_state['cpu']) - if torch.cuda.is_available(): - torch.cuda.random.set_rng_state_all(self.rng_state['cuda']) - self.need_load_rng_state = False - else: - self.logger.info( - 'Random state cannot be found in checkpoint file, ' - 'this may cause a random data order or model initialization.' - ) - - @staticmethod - def _restore_training_state(trainer, meta): - trainer._epoch = meta.get('epoch', trainer._epoch) - trainer._iter = meta.get('iter', trainer._iter) - trainer._inner_iter = meta.get('inner_iter', trainer._inner_iter) - - i = 0 - for hook in trainer.hooks: - if hasattr(hook, 'load_state_dict') and getattr( - hook, '_should_save', True): - key = f'{hook.__class__}-{i}' - if key in meta: - hook.load_state_dict(meta.get(key, {})) - else: - trainer.logger.warning( - f'The state_dict of hook {hook.__class__} at index {i} is not found in the checkpoint file.' - ) - i += 1 - - @classmethod - def load_checkpoint(cls, - filename, - trainer, - load_all_state=True, - strict=False): - """A static method to load checkpoint files. - - Args: - filename(str): An absolute model bin file(pth or bin) or a dir path with a file prefix(like epoch_1). - trainer(`EpochBasedTrainer`): The trainer instance. - load_all_state(`bool`): Load all states including the trainer states. - strict(`bool`): Load module state dict strictly. - - Returns: - A dict containing the train states saved by `_create_training_state` - """ - meta = cls().load_checkpoints(filename, trainer, load_all_state, - strict) - if load_all_state: - cls._restore_training_state(trainer, meta) - - if meta is not None: - _version = meta.get('modelscope') - if _version is not None and version.parse( - _version) < version.parse( - LoadCheckpointHook._TWO_PTH_FILE_VERSION): - trainer.logger.warning( - 'The unique pth file is split into a model file and ' - f'a trainer file since version {LoadCheckpointHook._TWO_PTH_FILE_VERSION},' - 'consider re-training your model or ' - 'using a converting script to split the single pth file into two.' - ) - trainer.logger.info( - f'Checkpoint {filename} saving time: {meta.get("time")}, modelscope version: {_version}' - ) - return meta - - @staticmethod - def load_trainer_state(trainer, train_state_file, load_all_state): - """Load trainer state file. - """ - - optimizer = getattr(trainer, 'optimizer', - None) if load_all_state else None - lr_scheduler = getattr(trainer, 'lr_scheduler', - None) if load_all_state else None - return load_checkpoint(train_state_file, None, optimizer, lr_scheduler) - - def load_model_state(self, trainer, model_file, strict): - """Load model state file. - """ - return load_checkpoint(model_file, - trainer.unwrap_module(trainer.model), None, - None) - - @Hook.overload_func(name='LoadCheckpointHook.load_checkpoints') - def load_checkpoints(self, checkpoint_path_prefix, trainer, load_all_state, - strict): - """Load checkpoint files of trainer state and model state. - - This is a strategic function which can be registered by other hook's function. - - Args: - checkpoint_path_prefix(str): The checkpoint dir with prefix or a model state file. - Example: '/tmp/test/epoch_0' or '/tmp/test/epoch_0.pth' - trainer(`EpochBasedTrainer`): The trainer instance. - load_all_state(`boolean`): Load all states (else load only module states). - strict(`boolean`): If strict, any unmatched keys will cause an error. - - Returns: - The meta info in json. - """ - _model_file, _train_state_file = _get_state_file_name( - checkpoint_path_prefix) - meta = {} - if os.path.isfile(_train_state_file): - meta = self.load_trainer_state(trainer, _train_state_file, - load_all_state) - else: - print(f'No trainer state file {_train_state_file} found, skip.') - self.load_model_state(trainer, _model_file, strict) - return meta - - -def _get_state_file_name(checkpoint_path_prefix): - """Get the default file name for state files. - - If the input is a checkpoint dir with prefix, this function will append suffix for both checkpoint files. - If the input is an absolute file name, this function will return it as the model file name, and append - suffix for the trainer file name. - - NOTE: a best checkpoint filename with float or int metric value inside - will not be judged as having a extension file name. like: '/tmp/test/epoch_0_accuracy0.85' - - Args: - checkpoint_path_prefix(`str`): The checkpoint dir with prefix or a model state file with extension file name. - like: '/tmp/test/epoch_0' - - Returns: - A tuple of model state file name and trainer state file name. - """ - base, ext = os.path.splitext(checkpoint_path_prefix) - if len(ext) == 0 or re.match(r'^\d+$', ext[1:]): - return checkpoint_path_prefix + CheckpointHook.MODEL_STATE_SUFFIX, \ - checkpoint_path_prefix + CheckpointHook.TRAINER_STATE_SUFFIX - else: - return checkpoint_path_prefix, base + CheckpointHook.TRAINER_STATE_SUFFIX.split( - '.')[0] + '.' + ext[1:] diff --git a/modelscope/trainers/hooks/compression/sparsity_hook.py b/modelscope/trainers/hooks/compression/sparsity_hook.py index 993488d8..e71c269a 100644 --- a/modelscope/trainers/hooks/compression/sparsity_hook.py +++ b/modelscope/trainers/hooks/compression/sparsity_hook.py @@ -1,7 +1,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import os -from modelscope import __version__ from modelscope.metainfo import Hooks from modelscope.trainers.hooks.builder import HOOKS from modelscope.trainers.hooks.hook import Hook diff --git a/modelscope/trainers/hooks/distributed/__init__.py b/modelscope/trainers/hooks/distributed/__init__.py new file mode 100644 index 00000000..17b9c429 --- /dev/null +++ b/modelscope/trainers/hooks/distributed/__init__.py @@ -0,0 +1,3 @@ +from .ddp_hook import DDPHook +from .deepspeed_hook import DeepspeedHook +from .megatron_hook import MegatronHook diff --git a/modelscope/trainers/hooks/ddp_hook.py b/modelscope/trainers/hooks/distributed/ddp_hook.py similarity index 89% rename from modelscope/trainers/hooks/ddp_hook.py rename to modelscope/trainers/hooks/distributed/ddp_hook.py index eaae2d89..2bdbe939 100644 --- a/modelscope/trainers/hooks/ddp_hook.py +++ b/modelscope/trainers/hooks/distributed/ddp_hook.py @@ -1,11 +1,11 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from modelscope.metainfo import Hooks +from modelscope.trainers.hooks.builder import HOOKS +from modelscope.trainers.hooks.hook import Hook +from modelscope.trainers.hooks.priority import Priority from modelscope.utils.constant import DistributedParallelType from modelscope.utils.device import create_device from modelscope.utils.torch_utils import get_local_rank, init_dist -from .builder import HOOKS -from .hook import Hook -from .priority import Priority @HOOKS.register_module(module_name=Hooks.DDPHook) diff --git a/modelscope/trainers/hooks/deepspeed_hook.py b/modelscope/trainers/hooks/distributed/deepspeed_hook.py similarity index 64% rename from modelscope/trainers/hooks/deepspeed_hook.py rename to modelscope/trainers/hooks/distributed/deepspeed_hook.py index a34b3f6f..7dddc5d9 100644 --- a/modelscope/trainers/hooks/deepspeed_hook.py +++ b/modelscope/trainers/hooks/distributed/deepspeed_hook.py @@ -8,72 +8,48 @@ from deepspeed import DeepSpeedEngine from megatron_util import mpu, print_rank_0 from modelscope.metainfo import Hooks +from modelscope.trainers.hooks import LoadCheckpointHook from modelscope.trainers.hooks.builder import HOOKS +from modelscope.trainers.hooks.checkpoint.checkpoint_hook import ( + BestCkptSaverHook, CheckpointHook) from modelscope.trainers.hooks.hook import Hook from modelscope.trainers.hooks.priority import Priority from modelscope.utils.checkpoint import save_checkpoint from modelscope.utils.logger import get_logger -from .checkpoint_hook import CheckpointHook, LoadCheckpointHook -from .megatron_hook import MegatronHook +from ..checkpoint.checkpoint_processor import CheckpointProcessor +from ..lr_scheduler_hook import LrSchedulerProcessor +from ..optimizer.base import OptimizerHook, OptimizerProcessor -@HOOKS.register_module(module_name=Hooks.DeepspeedHook) -class DeepspeedHook(MegatronHook): - PRIORITY = Priority.VERY_HIGH +class DeepspeedProcessor(CheckpointProcessor, LrSchedulerProcessor, + OptimizerProcessor): - def __init__(self, - deepspeed_activation_checkpointing=True, - save_zero_checkpoint=False, - with_mpu=True): - self.save_zero_checkpoint = save_zero_checkpoint - self.deepspeed_activation_checkpointing = deepspeed_activation_checkpointing - # TODO without mpu - self.with_mpu = with_mpu - assert with_mpu, 'DeepspeedHook now is only for mpu models.' + _BIN_FILE_DIR = 'model' - def register_strategy(self): - Hook.overload(name='OptimizerHook.backward', function=self.backward) - Hook.overload( - name='OptimizerHook.initialize_optimizer', function=self.idle) - Hook.overload(name='LrSchedulerHook.step', function=self.idle) - Hook.overload( - name='CheckpointHook.save_checkpoints', - function=self.save_checkpoints) - Hook.overload( - name='LoadCheckpointHook.load_checkpoints', - function=self.load_checkpoints) - Hook.overload( - name='CheckpointHook.remove_checkpoints', - function=self.remove_checkpoints) - Hook.overload( - name='CheckpointHook.prepare_output', function=self.prepare_output) - if self.with_mpu: - Hook.overload( - name='CheckpointHook.should_save_on_rank', - function=self.should_save_on_rank) + def rank_name(self): + # TODO + try: + tp_world_size = mpu.get_tensor_model_parallel_world_size() + if tp_world_size == 1: + return '' + mp_rank = mpu.get_tensor_model_parallel_rank() + return '_mp_rank_{:02d}'.format(mp_rank) + except (ImportError, AssertionError): + return '' - def backward(self, trainer, loss_keys, cumulative_iters, grad_clip): - # assert cumulative_iters == 1, 'DeepSpeed only support cumulative_iters=1' - # The `trainer.model` here is actually a deepspeed engine object. - # backward step - for k in loss_keys: - loss = trainer.train_outputs[k] - trainer.model.backward(loss) - - # update parameters - trainer.model.step() - - def idle(self, *args, **kwargs): - pass + def get_bin_file(self): + mp_rank = mpu.get_tensor_model_parallel_rank() + rank = '{:02d}'.format(mp_rank) + return f'mp_rank_{rank}_model_states.pt' def save_checkpoints(self, trainer, checkpoint_path_prefix, - output_sub_dir, + output_dir, meta=None): model = trainer.unwrap_module(trainer.model) _train_state_file = checkpoint_path_prefix + self.rank_name( - ) + CheckpointHook.TRAINER_STATE_SUFFIX + ) + CheckpointProcessor.TRAINER_STATE_SUFFIX # Save pth file without model state_dict save_checkpoint( model, _train_state_file, None, None, meta=meta, with_model=False) @@ -84,16 +60,22 @@ class DeepspeedHook(MegatronHook): bin_file = self.get_bin_file() src_file = os.path.join(checkpoint_path_prefix, bin_file) - dest_file = os.path.join(save_dir, output_sub_dir, self._BIN_FILE_DIR, - bin_file) + dest_file = os.path.join(output_dir, self._BIN_FILE_DIR, bin_file) if os.path.isfile(dest_file): os.unlink(dest_file) - os.link(src_file, dest_file) + try: + os.link(src_file, dest_file) + except OSError as e: + get_logger().error( + f'Link {src_file} to {dest_file} error: {e}, ' + 'changing to copy the bin file, this may case more space usage.' + ) + shutil.copyfile(src_file, dest_file) def remove_checkpoints(self, trainer, checkpoint_path_prefix): _train_state_file = checkpoint_path_prefix + self.rank_name( - ) + CheckpointHook.TRAINER_STATE_SUFFIX + ) + CheckpointProcessor.TRAINER_STATE_SUFFIX if os.path.isfile(_train_state_file): os.remove(_train_state_file) @@ -107,10 +89,10 @@ class DeepspeedHook(MegatronHook): meta = {} _train_state_file = checkpoint_path_prefix + self.rank_name( - ) + CheckpointHook.TRAINER_STATE_SUFFIX + ) + CheckpointProcessor.TRAINER_STATE_SUFFIX if os.path.isfile(_train_state_file): - meta = LoadCheckpointHook.load_trainer_state( - trainer, _train_state_file, load_all_state) + meta = self.load_trainer_state(trainer, _train_state_file, + load_all_state) if isinstance(trainer.model, DeepSpeedEngine): # DeepSpeedEngine is initialized @@ -138,6 +120,57 @@ class DeepspeedHook(MegatronHook): checkpoint, strict=strict) return meta + def backward(self, trainer, loss_keys, cumulative_iters, grad_clip): + # assert cumulative_iters == 1, 'DeepSpeed only support cumulative_iters=1' + # The `trainer.model` here is actually a deepspeed engine object. + # backward step + for k in loss_keys: + loss = trainer.train_outputs[k] + trainer.model.backward(loss) + + # update parameters + trainer.model.step() + + def initialize_optimizer(self, trainer): + pass + + def step(self, trainer): + pass + + +@HOOKS.register_module(module_name=Hooks.DeepspeedHook) +class DeepspeedHook(Hook): + PRIORITY = Priority.VERY_HIGH + + def __init__(self, + deepspeed_activation_checkpointing=True, + save_zero_checkpoint=False, + with_mpu=True): + self.save_zero_checkpoint = save_zero_checkpoint + self.deepspeed_activation_checkpointing = deepspeed_activation_checkpointing + # TODO without mpu + self.with_mpu = with_mpu + assert with_mpu, 'DeepspeedHook now is only for mpu models.' + + def register_processor(self, trainer): + processor = DeepspeedProcessor() + optimizer_hook = trainer.get_hook(OptimizerHook) + if len(optimizer_hook) > 0 and not isinstance( + optimizer_hook[0].processor, DeepspeedProcessor): + optimizer_hook[0].set_processor(processor) + ckpt_hook = trainer.get_hook(CheckpointHook) + if len(ckpt_hook) > 0 and not isinstance(ckpt_hook[0].processor, + DeepspeedProcessor): + ckpt_hook[0].set_processor(processor) + best_ckpt_hook = trainer.get_hook(BestCkptSaverHook) + if len(best_ckpt_hook) > 0 and not isinstance( + best_ckpt_hook[0].processor, DeepspeedProcessor): + best_ckpt_hook[0].set_processor(processor) + load_ckpt_hook = trainer.get_hook(LoadCheckpointHook) + if len(load_ckpt_hook) > 0 and not isinstance( + load_ckpt_hook[0].processor, DeepspeedProcessor): + load_ckpt_hook[0].set_processor(processor) + def before_val(self, trainer): pass diff --git a/modelscope/trainers/hooks/megatron_hook.py b/modelscope/trainers/hooks/distributed/megatron_hook.py similarity index 70% rename from modelscope/trainers/hooks/megatron_hook.py rename to modelscope/trainers/hooks/distributed/megatron_hook.py index f01288de..c4aeaf19 100644 --- a/modelscope/trainers/hooks/megatron_hook.py +++ b/modelscope/trainers/hooks/distributed/megatron_hook.py @@ -1,19 +1,129 @@ import os -from copy import deepcopy +import shutil import torch from megatron_util import mpu from modelscope.metainfo import Hooks +from modelscope.trainers import EpochBasedTrainer from modelscope.trainers.hooks.builder import HOOKS +from modelscope.trainers.hooks.checkpoint.checkpoint_hook import ( + BestCkptSaverHook, CheckpointHook, CheckpointProcessor) +from modelscope.trainers.hooks.checkpoint.load_checkpoint_hook import \ + LoadCheckpointHook from modelscope.trainers.hooks.hook import Hook -from modelscope.trainers.parallel.builder import build_parallel from modelscope.utils.checkpoint import load_checkpoint, save_checkpoint from modelscope.utils.constant import DistributedParallelType from modelscope.utils.device import create_device +from modelscope.utils.logger import get_logger from modelscope.utils.megatron_utils import is_megatron_initialized from modelscope.utils.torch_utils import get_local_rank -from .checkpoint_hook import CheckpointHook, LoadCheckpointHook + + +class MpuProcessor(CheckpointProcessor): + + _BIN_FILE_DIR = 'model' + + def rank_name(self): + # TODO + try: + tp_world_size = mpu.get_tensor_model_parallel_world_size() + if tp_world_size == 1: + return '' + mp_rank = mpu.get_tensor_model_parallel_rank() + return '_mp_rank_{:02d}'.format(mp_rank) + except (ImportError, AssertionError): + return '' + + def get_bin_file(self): + mp_rank = mpu.get_tensor_model_parallel_rank() + rank = '{:02d}'.format(mp_rank) + return f'mp_rank_{rank}_model_states.pt' + + def should_save_on_rank(self, trainer): + # TODO + return (not torch.distributed.is_initialized() + ) or mpu.get_data_parallel_rank() == 0 + + def prepare_output(self, trainer, output_dir): + config = trainer.cfg + CheckpointProcessor.copy_files_and_dump_config(trainer, output_dir, + config, + self._BIN_FILE_DIR) + os.makedirs( + os.path.join(output_dir, self._BIN_FILE_DIR), exist_ok=True) + + def save_checkpoints(self, + trainer, + checkpoint_path_prefix, + output_dir, + meta=None): + model = trainer.unwrap_module(trainer.model) + _train_state_file = checkpoint_path_prefix + self.rank_name( + ) + CheckpointProcessor.TRAINER_STATE_SUFFIX + # Save pth file without model state_dict + save_checkpoint( + model, + _train_state_file, + trainer.optimizer, + trainer.lr_scheduler, + meta=meta, + with_model=False) + + save_dir = os.path.dirname(checkpoint_path_prefix) + prefix = os.path.basename(checkpoint_path_prefix) + bin_file = self.get_bin_file() + prefix_bin_file = os.path.join(save_dir, prefix + '_' + bin_file) + save_checkpoint(model, prefix_bin_file, with_meta=False) + + src_file = prefix_bin_file + dest_file = os.path.join(output_dir, self._BIN_FILE_DIR, bin_file) + if os.path.isfile(dest_file): + os.unlink(dest_file) + + try: + os.link(src_file, dest_file) + except OSError as e: + get_logger().error( + f'Link {src_file} to {dest_file} error: {e}, ' + 'changing to copy the bin file, this may case more space usage.' + ) + shutil.copyfile(src_file, dest_file) + + def remove_checkpoints(self, trainer, checkpoint_path_prefix): + _train_state_file = checkpoint_path_prefix + self.rank_name( + ) + CheckpointProcessor.TRAINER_STATE_SUFFIX + if os.path.isfile(_train_state_file): + os.remove(_train_state_file) + + save_dir = os.path.dirname(checkpoint_path_prefix) + prefix = os.path.basename(checkpoint_path_prefix) + bin_file = self.get_bin_file() + absolute_file = os.path.join(save_dir, prefix + '_' + bin_file) + if os.path.isfile(absolute_file): + os.remove(absolute_file) + + def load_checkpoints(self, checkpoint_path_prefix, trainer, load_all_state, + strict): + model = trainer.unwrap_module(trainer.model) + if os.path.isdir(checkpoint_path_prefix): + save_dir = checkpoint_path_prefix + bin_file = self.get_bin_file() + model_file = os.path.join(save_dir, bin_file) + load_checkpoint(model_file, model, None, None) + else: + _train_state_file = checkpoint_path_prefix + self.rank_name( + ) + CheckpointProcessor.TRAINER_STATE_SUFFIX + meta = LoadCheckpointHook.load_trainer_state( + trainer, _train_state_file, load_all_state) + + save_dir = os.path.dirname(checkpoint_path_prefix) + prefix = os.path.basename(checkpoint_path_prefix) + bin_file = self.get_bin_file() + + model_file = os.path.join(save_dir, prefix + '_' + bin_file) + load_checkpoint(model_file, model, None, None) + return meta @HOOKS.register_module(module_name=Hooks.MegatronHook) @@ -24,21 +134,20 @@ class MegatronHook(Hook): def __init__(self): self.wrapped = False - def register_strategy(self): - Hook.overload( - name='CheckpointHook.should_save_on_rank', - function=self.should_save_on_rank) - Hook.overload( - name='CheckpointHook.save_checkpoints', - function=self.save_checkpoints) - Hook.overload( - name='LoadCheckpointHook.load_checkpoints', - function=self.load_checkpoints) - Hook.overload( - name='CheckpointHook.remove_checkpoints', - function=self.remove_checkpoints) - Hook.overload( - name='CheckpointHook.prepare_output', function=self.prepare_output) + def register_processor(self, trainer: EpochBasedTrainer): + processor = MpuProcessor() + ckpt_hook = trainer.get_hook(CheckpointHook) + if len(ckpt_hook) > 0 and not isinstance(ckpt_hook[0].processor, + MpuProcessor): + ckpt_hook[0].set_processor(processor) + best_ckpt_hook = trainer.get_hook(BestCkptSaverHook) + if len(best_ckpt_hook) > 0 and not isinstance( + best_ckpt_hook[0].processor, MpuProcessor): + best_ckpt_hook[0].set_processor(processor) + load_ckpt_hook = trainer.get_hook(LoadCheckpointHook) + if len(load_ckpt_hook) > 0 and not isinstance( + load_ckpt_hook[0].processor, MpuProcessor): + load_ckpt_hook[0].set_processor(processor) def after_init(self, trainer): assert is_megatron_initialized() @@ -63,97 +172,3 @@ class MegatronHook(Hook): if not self.wrapped: trainer.model = trainer.to_parallel(trainer.model) self.wrapped = True - - def should_save_on_rank(self, trainer): - # TODO - return (not torch.distributed.is_initialized() - ) or mpu.get_data_parallel_rank() == 0 - - def rank_name(self): - # TODO - try: - tp_world_size = mpu.get_tensor_model_parallel_world_size() - if tp_world_size == 1: - return '' - mp_rank = mpu.get_tensor_model_parallel_rank() - return '_mp_rank_{:02d}'.format(mp_rank) - except (ImportError, AssertionError): - return '' - - def get_bin_file(self): - mp_rank = mpu.get_tensor_model_parallel_rank() - rank = '{:02d}'.format(mp_rank) - return f'mp_rank_{rank}_model_states.pt' - - def save_checkpoints(self, - trainer, - checkpoint_path_prefix, - output_sub_dir, - meta=None): - model = trainer.unwrap_module(trainer.model) - _train_state_file = checkpoint_path_prefix + self.rank_name( - ) + CheckpointHook.TRAINER_STATE_SUFFIX - # Save pth file without model state_dict - save_checkpoint( - model, - _train_state_file, - trainer.optimizer, - trainer.lr_scheduler, - meta=meta, - with_model=False) - - save_dir = os.path.dirname(checkpoint_path_prefix) - prefix = os.path.basename(checkpoint_path_prefix) - bin_file = self.get_bin_file() - prefix_bin_file = os.path.join(save_dir, prefix + '_' + bin_file) - save_checkpoint(model, prefix_bin_file, with_meta=False) - - src_file = prefix_bin_file - dest_file = os.path.join(save_dir, output_sub_dir, self._BIN_FILE_DIR, - bin_file) - if os.path.isfile(dest_file): - os.unlink(dest_file) - - os.link(src_file, dest_file) - - def remove_checkpoints(self, trainer, checkpoint_path_prefix): - _train_state_file = checkpoint_path_prefix + self.rank_name( - ) + CheckpointHook.TRAINER_STATE_SUFFIX - if os.path.isfile(_train_state_file): - os.remove(_train_state_file) - - save_dir = os.path.dirname(checkpoint_path_prefix) - prefix = os.path.basename(checkpoint_path_prefix) - bin_file = self.get_bin_file() - absolute_file = os.path.join(save_dir, prefix + '_' + bin_file) - if os.path.isfile(absolute_file): - os.remove(absolute_file) - - def load_checkpoints(self, checkpoint_path_prefix, trainer, load_all_state, - strict): - model = trainer.unwrap_module(trainer.model) - if os.path.isdir(checkpoint_path_prefix): - save_dir = checkpoint_path_prefix - bin_file = self.get_bin_file() - model_file = os.path.join(save_dir, bin_file) - load_checkpoint(model_file, model, None, None) - else: - _train_state_file = checkpoint_path_prefix + self.rank_name( - ) + CheckpointHook.TRAINER_STATE_SUFFIX - meta = LoadCheckpointHook.load_trainer_state( - trainer, _train_state_file, load_all_state) - - save_dir = os.path.dirname(checkpoint_path_prefix) - prefix = os.path.basename(checkpoint_path_prefix) - bin_file = self.get_bin_file() - - model_file = os.path.join(save_dir, prefix + '_' + bin_file) - load_checkpoint(model_file, model, None, None) - return meta - - def prepare_output(self, trainer, output_dir): - config = trainer.cfg - CheckpointHook.copy_files_and_dump_config(trainer, output_dir, config, - self._BIN_FILE_DIR) - os.makedirs( - os.path.join(output_dir, self._BIN_FILE_DIR), exist_ok=True) diff --git a/modelscope/trainers/hooks/early_stop_hook.py b/modelscope/trainers/hooks/early_stop_hook.py index b15e8e5a..7aba69a4 100644 --- a/modelscope/trainers/hooks/early_stop_hook.py +++ b/modelscope/trainers/hooks/early_stop_hook.py @@ -9,6 +9,12 @@ from .hook import Hook from .priority import Priority +class EarlyStopStrategy: + by_epoch = 'by_epoch' + by_step = 'by_step' + no = 'no' + + @HOOKS.register_module(module_name=Hooks.EarlyStopHook) class EarlyStopHook(Hook): """Early stop when a specific metric stops improving. @@ -16,14 +22,13 @@ class EarlyStopHook(Hook): Args: metric_key (str): Metric key to be monitored. rule (str): Comparison rule for best score. Support "max" and "min". - If rule is "max", the training will stop when `metric_key` has stopped increaing. + If rule is "max", the training will stop when `metric_key` has stopped increasing. If rule is "min", the training will stop when `metric_key` has stopped decreasing. patience (int): Trainer will stop if the monitored metric did not improve for the last `patience` times. - min_delta (float): Minimum change in the monitored metric to quailfy as an improvement. + min_delta (float): Minimum change in the monitored metric to qualify as an improvement. check_finite (bool): If true, stops training when the metric becomes NaN or infinite. - by_epoch (int): Saving checkpoints by epoch or by iteration. - interval (int): The frequency to trigger early stop check. If `by_epoch=True`, - it means the number of epochs, else means the number of iterations. + early_stop_strategy (str): The strategy to early stop, can be by_epoch/by_step/none + interval (int): The frequency to trigger early stop check, by epoch or step. """ PRIORITY = Priority.VERY_LOW @@ -35,14 +40,19 @@ class EarlyStopHook(Hook): patience: int = 3, min_delta: float = 0.0, check_finite: bool = True, - by_epoch: bool = True, - interval: int = 1): + early_stop_strategy: str = EarlyStopStrategy.by_epoch, + interval: int = 1, + **kwargs): self.metric_key = metric_key self.rule = rule self.patience = patience self.min_delta = min_delta self.check_finite = check_finite - self.by_epoch = by_epoch + if 'by_epoch' in kwargs: + self.early_stop_strategy = EarlyStopStrategy.by_epoch if kwargs[ + 'by_epoch'] else EarlyStopStrategy.by_step + else: + self.early_stop_strategy = early_stop_strategy self.interval = interval self.wait_count = 0 @@ -89,7 +99,7 @@ class EarlyStopHook(Hook): trainer._stop_training = True def after_train_epoch(self, trainer): - if not self.by_epoch: + if self.early_stop_strategy != EarlyStopStrategy.by_epoch: return if not self.every_n_epochs(trainer, self.interval): @@ -99,7 +109,7 @@ class EarlyStopHook(Hook): self._stop_training(trainer) def after_train_iter(self, trainer): - if self.by_epoch: + if self.early_stop_strategy != EarlyStopStrategy.by_step: return if not self.every_n_iters(trainer, self.interval): diff --git a/modelscope/trainers/hooks/evaluation_hook.py b/modelscope/trainers/hooks/evaluation_hook.py index 80c8c31a..c29a6d6a 100644 --- a/modelscope/trainers/hooks/evaluation_hook.py +++ b/modelscope/trainers/hooks/evaluation_hook.py @@ -1,11 +1,18 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from collections import OrderedDict +from typing import Optional from modelscope.metainfo import Hooks from .builder import HOOKS from .hook import Hook +class EvaluationStrategy: + by_epoch = 'by_epoch' + by_step = 'by_step' + no = 'no' + + @HOOKS.register_module(module_name=Hooks.EvaluationHook) class EvaluationHook(Hook): """ @@ -18,21 +25,34 @@ class EvaluationHook(Hook): Default: None, validate every interval epochs/iterations from scratch. """ - def __init__(self, interval=1, by_epoch=True, start_idx=None): + def __init__(self, + interval: Optional[int] = 1, + eval_strategy: Optional[str] = EvaluationStrategy.by_epoch, + start_idx: Optional[int] = None, + **kwargs): assert interval > 0, 'interval must be a positive number' self.interval = interval self.start_idx = start_idx - self.by_epoch = by_epoch + self.last_eval_tag = (None, None) + if 'by_epoch' in kwargs: + self.eval_strategy = EvaluationStrategy.by_epoch if kwargs[ + 'by_epoch'] else EvaluationStrategy.by_step + else: + self.eval_strategy = eval_strategy def after_train_iter(self, trainer): """Called after every training iter to evaluate the results.""" - if not self.by_epoch and self._should_evaluate(trainer): + if self.eval_strategy == EvaluationStrategy.by_step and self._should_evaluate( + trainer): self.do_evaluate(trainer) + self.last_eval_tag = ('iter', trainer.iter) def after_train_epoch(self, trainer): """Called after every training epoch to evaluate the results.""" - if self.by_epoch and self._should_evaluate(trainer): + if self.eval_strategy == EvaluationStrategy.by_epoch and self._should_evaluate( + trainer): self.do_evaluate(trainer) + self.last_eval_tag = ('epoch', trainer.epoch) def add_visualization_info(self, trainer, results): if trainer.visualization_buffer.output.get('eval_results', @@ -64,7 +84,7 @@ class EvaluationHook(Hook): Returns: bool: The flag indicating whether to perform evaluation. """ - if self.by_epoch: + if self.eval_strategy == EvaluationStrategy.by_epoch: current = trainer.epoch check_time = self.every_n_epochs else: diff --git a/modelscope/trainers/hooks/hook.py b/modelscope/trainers/hooks/hook.py index 70e06fbd..93ea8541 100644 --- a/modelscope/trainers/hooks/hook.py +++ b/modelscope/trainers/hooks/hook.py @@ -22,9 +22,6 @@ class Hook: PRIORITY = Priority.NORMAL - # The strategic function dict. - _strategies = dict() - def after_init(self, trainer): """ Will be called at the end of the trainer's `__init__` method @@ -201,42 +198,48 @@ class Hook: """ self.after_iter(trainer) - def every_n_epochs(self, trainer, n): + @staticmethod + def every_n_epochs(trainer, n): """ Whether to reach every ``n`` epochs Returns: bool """ return (trainer.epoch + 1) % n == 0 if n > 0 else False - def every_n_inner_iters(self, runner, n): + @staticmethod + def every_n_inner_iters(runner, n): """ Whether to reach every ``n`` iterations at every epoch Returns: bool """ return (runner.inner_iter + 1) % n == 0 if n > 0 else False - def every_n_iters(self, trainer, n): + @staticmethod + def every_n_iters(trainer, n): """ Whether to reach every ``n`` iterations Returns: bool """ return (trainer.iter + 1) % n == 0 if n > 0 else False - def end_of_epoch(self, trainer): + @staticmethod + def end_of_epoch(trainer): """ Whether to reach the end of every epoch Returns: bool """ return trainer.inner_iter + 1 == trainer.iters_per_epoch - def is_last_epoch(self, trainer): + @staticmethod + def is_last_epoch(trainer): """ Whether to reach the last epoch Returns: bool """ return trainer.epoch + 1 == trainer.max_epochs - def is_last_iter(self, trainer): + @staticmethod + def is_last_iter(trainer): """ Whether to reach the last iteration in the entire training process Returns: bool @@ -256,54 +259,3 @@ class Hook: def load_state_dict(self, state_dict): pass - - @staticmethod - def clear_strategies(): - Hook._strategies.clear() - - @staticmethod - def overload(function, name=None): - """Register a function to a strategic function. - - Args: - function(`method` or `Callable`): The function instance. - name(`str`): The name of the strategic function, which specifies by the method `consume` - """ - - _name = name or function.__name__ - if _name not in Hook._strategies: - Hook._strategies[_name] = [] - - Hook._strategies[_name].append(function) - - @staticmethod - def overload_func(name=None): - """Declare a function as a strategic function, which can be replaced by some other functions. - - This function should be used in annotations. - - Args: - name(str): The strategic function name. - """ - - def _register(function): - - @wraps(function) - def _call(*args, **kwargs): - _name = name or function.__name__ - producers = Hook._strategies.get(_name, []) - - if len(producers) == 0: - return function(*args, **kwargs) - else: - if len(producers) > 1: - raise ValueError( - f'Multiple functions registered to {_name}, ' - f'here is the list: {producers}') - if isinstance(args[0], Hook): - args = args[1:] - return producers[0](*args, **kwargs) - - return _call - - return _register diff --git a/modelscope/trainers/hooks/lr_scheduler_hook.py b/modelscope/trainers/hooks/lr_scheduler_hook.py index 28ce250c..51a8e858 100644 --- a/modelscope/trainers/hooks/lr_scheduler_hook.py +++ b/modelscope/trainers/hooks/lr_scheduler_hook.py @@ -1,4 +1,5 @@ # Copyright (c) Alibaba, Inc. and its affiliates. + from modelscope.metainfo import Hooks from modelscope.trainers.lrscheduler.builder import build_lr_scheduler from modelscope.utils.constant import LogKeys @@ -9,6 +10,42 @@ from .hook import Hook from .priority import Priority +class LrSchedulerProcessor: + + def __init__(self): + self.lr_strategy = None + self.warmup_lr_scheduler = None + + def set_lr_strategy(self, lr_strategy): + self.lr_strategy = lr_strategy + + def set_warmup_lr_scheduler(self, warmup_lr_scheduler): + self.warmup_lr_scheduler = warmup_lr_scheduler + + def initialize_lr_scheduler(self, trainer): + """Initialize the lr scheduler. + + This is a strategic function which can be registered by other hook's function. + """ + pass + + def step(self, trainer): + """Do lr scheduler's step. + + This is a strategic function which can be registered by other hook's function. + """ + if self.warmup_lr_scheduler is not None: + self.warmup_lr_scheduler.step() + else: + trainer.lr_scheduler.step() + + +class LrStrategy: + by_epoch = 'by_epoch' + by_step = 'by_step' + no = 'no' + + @HOOKS.register_module(module_name=Hooks.LrSchedulerHook) class LrSchedulerHook(Hook): """Lr scheduler. @@ -19,38 +56,33 @@ class LrSchedulerHook(Hook): """ PRIORITY = Priority.LOW - def __init__(self, by_epoch=True, warmup=None, **kwargs) -> None: + def __init__(self, + lr_strategy=LrStrategy.by_epoch, + warmup=None, + **kwargs) -> None: super().__init__() - self.by_epoch = by_epoch + if 'by_epoch' in kwargs: + self.lr_strategy = LrStrategy.by_epoch if kwargs[ + 'by_epoch'] else LrStrategy.by_step + else: + self.lr_strategy = lr_strategy self.warmup = warmup self.warmup_lr_scheduler = None + self.processor = LrSchedulerProcessor() + + def set_processor(self, processor): + self.processor = processor def before_run(self, trainer): - self.initialize_lr_scheduler(trainer) + self.processor.set_lr_strategy(self.lr_strategy) if self.warmup is not None: assert isinstance(self.warmup, dict) and 'type' in self.warmup self.warmup_lr_scheduler = build_lr_scheduler( cfg=self.warmup, default_args={'base_scheduler': trainer.lr_scheduler}) + self.processor.set_warmup_lr_scheduler(self.warmup_lr_scheduler) - @Hook.overload_func(name='LrSchedulerHook.initialize_lr_scheduler') - def initialize_lr_scheduler(self, trainer): - """Initialize the lr scheduler. - - This is a strategic function which can be registered by other hook's function. - """ - pass - - @Hook.overload_func(name='LrSchedulerHook.step') - def step(self, trainer): - """Do lr scheduler's step. - - This is a strategic function which can be registered by other hook's function. - """ - if self.warmup_lr_scheduler is not None: - self.warmup_lr_scheduler.step() - else: - trainer.lr_scheduler.step() + self.processor.initialize_lr_scheduler(trainer) def get_current_lr(self, trainer): import torch @@ -67,17 +99,17 @@ class LrSchedulerHook(Hook): return lr def after_train_iter(self, trainer): - if not self.by_epoch and trainer.iter >= getattr( + if self.lr_strategy == LrStrategy.by_step and trainer.iter >= getattr( trainer, 'cumulative_iters', 1) - 1: - self.step(trainer) + self.processor.step(trainer) trainer.log_buffer.output[LogKeys.LR] = self._get_log_lr(trainer) def before_train_epoch(self, trainer): trainer.log_buffer.output[LogKeys.LR] = self._get_log_lr(trainer) def after_train_epoch(self, trainer): - if self.by_epoch: - self.step(trainer) + if self.lr_strategy == LrStrategy.by_epoch: + self.processor.step(trainer) def _get_log_lr(self, trainer): cur_lr = self.get_current_lr(trainer) @@ -94,6 +126,29 @@ class LrSchedulerHook(Hook): return lr +class PlateauLrSchedulerProcessor(LrSchedulerProcessor): + + def __init__(self, metric_key): + super().__init__() + self.metric_key = metric_key + + def step(self, trainer): + # adapt to evaluation interval is greater than 1 + if trainer.metric_values is None: + if is_master(): + print( + f'Current epoch {trainer.epoch} has no evaluation metric values, skip lr_scheduler.step() !' + ) + return + + metrics = trainer.metric_values[self.metric_key] + if self.lr_strategy == LrStrategy.by_epoch: + if self.warmup_lr_scheduler is not None: + self.warmup_lr_scheduler.step(metrics=metrics) + else: + trainer.lr_scheduler.step(metrics=metrics) + + @HOOKS.register_module(module_name=Hooks.PlateauLrSchedulerHook) class PlateauLrSchedulerHook(Hook): """Lr scheduler hook for `ReduceLROnPlateau`. @@ -105,10 +160,16 @@ class PlateauLrSchedulerHook(Hook): PRIORITY = Priority.LOW # should be after EvaluationHook def __init__(self, metric_key, **kwargs): + super().__init__() self.metric_key = metric_key - def register_strategy(self): - Hook.overload(name='LrSchedulerHook.step', function=self.step) + def register_processor(self, trainer): + lr_scheduler_hook = trainer.get_hook(LrSchedulerHook) + if len(lr_scheduler_hook) > 0 and type( + lr_scheduler_hook[0].processor) in (type(None), + LrSchedulerProcessor): + lr_scheduler_hook[0].set_processor( + PlateauLrSchedulerProcessor(self.metric_key)) def before_run(self, trainer): if not hasattr(trainer, 'logger'): @@ -116,23 +177,6 @@ class PlateauLrSchedulerHook(Hook): else: self.logger = trainer.logger - def step(self, trainer): - # adapt to evaluation intervel is greater than 1 - if trainer.metric_values is None: - if is_master(): - self.logger.warning( - f'Current epoch {trainer.epoch} has no evaluation metric values, skip lr_scheduler.step() !' - ) - return - - metrics = trainer.metric_values[self.metric_key] - lr_scheduler_hook = trainer.get_hook(LrSchedulerHook)[0] - if lr_scheduler_hook.by_epoch: - if lr_scheduler_hook.warmup_lr_scheduler is not None: - lr_scheduler_hook.warmup_lr_scheduler.step(metrics=metrics) - else: - trainer.lr_scheduler.step(metrics=metrics) - @HOOKS.register_module(module_name=Hooks.NoneLrSchedulerHook) class NoneLrSchedulerHook(LrSchedulerHook): diff --git a/modelscope/trainers/hooks/optimizer/apex_optimizer_hook.py b/modelscope/trainers/hooks/optimizer/apex_optimizer_hook.py index bd1034f3..3c874ccf 100644 --- a/modelscope/trainers/hooks/optimizer/apex_optimizer_hook.py +++ b/modelscope/trainers/hooks/optimizer/apex_optimizer_hook.py @@ -7,40 +7,14 @@ from packaging import version from modelscope.metainfo import Hooks from modelscope.trainers.hooks import Hook from modelscope.trainers.hooks.builder import HOOKS -from .base import OptimizerHook +from .base import OptimizerHook, OptimizerProcessor -@HOOKS.register_module(module_name=Hooks.ApexAMPOptimizerHook) -class ApexAMPOptimizerHook(Hook): - """ - Fp16 optimizer, if torch version is less than 1.6.0, - you must install apex (https://www.github.com/nvidia/apex) else use torch.cuda.amp by default +class ApexOptimizerProcessor(OptimizerProcessor): - Args: - opt_level (str): "O0" and "O3" are not true mixed precision, - but they are useful for establishing accuracy and speed baselines, respectively. - "O1" and "O2" are different implementations of mixed precision. - Try both, and see what gives the best speedup and accuracy for your model. - """ - - PRIORITY = OptimizerHook.PRIORITY - - def __init__(self, opt_level='O1', **kwargs): + def __init__(self, opt_level): self.opt_level = opt_level - try: - from apex import amp - except ImportError: - raise ValueError( - 'apex not installed, please install apex from https://www.github.com/nvidia/apex.' - ) - - def register_strategy(self): - Hook.overload( - name='OptimizerHook.initialize_optimizer', - function=self.initialize_optimizer) - Hook.overload(name='OptimizerHook.backward', function=self.backward) - def initialize_optimizer(self, trainer): from apex import amp @@ -68,10 +42,44 @@ class ApexAMPOptimizerHook(Hook): trainer.optimizer) as scaled_loss: scaled_loss.backward() - if self.every_n_iters(trainer, cumulative_iters): + if Hook.every_n_iters(trainer, cumulative_iters): if grad_clip is not None: - OptimizerHook.clip_grads(trainer.model.parameters(), - **grad_clip) + OptimizerProcessor.clip_grads(trainer.model.parameters(), + **grad_clip) trainer.optimizer.step() trainer.optimizer.zero_grad() + + +@HOOKS.register_module(module_name=Hooks.ApexAMPOptimizerHook) +class ApexAMPOptimizerHook(Hook): + """ + Fp16 optimizer, if torch version is less than 1.6.0, + you must install apex (https://www.github.com/nvidia/apex) else use torch.cuda.amp by default + + Args: + opt_level (str): "O0" and "O3" are not true mixed precision, + but they are useful for establishing accuracy and speed baselines, respectively. + "O1" and "O2" are different implementations of mixed precision. + Try both, and see what gives the best speedup and accuracy for your model. + """ + + PRIORITY = OptimizerHook.PRIORITY + + def __init__(self, opt_level='O1', **kwargs): + self.opt_level = opt_level + + try: + from apex import amp + except ImportError: + raise ValueError( + 'apex not installed, please install apex from https://www.github.com/nvidia/apex.' + ) + + def register_processor(self, trainer): + optimizer_hook = trainer.get_hook(OptimizerHook) + if len(optimizer_hook) > 0 and type( + optimizer_hook[0].processor) in (type(None), + OptimizerProcessor): + optimizer_hook[0].set_processor( + ApexOptimizerProcessor(self.opt_level)) diff --git a/modelscope/trainers/hooks/optimizer/base.py b/modelscope/trainers/hooks/optimizer/base.py index f0d62612..ca20720d 100644 --- a/modelscope/trainers/hooks/optimizer/base.py +++ b/modelscope/trainers/hooks/optimizer/base.py @@ -10,6 +10,48 @@ from modelscope.trainers.hooks.hook import Hook from modelscope.trainers.hooks.priority import Priority +class OptimizerProcessor: + + def initialize_optimizer(self, trainer): + """Initialize the optimizer. + + This is a strategic function which can be registered by other hook's function. + """ + trainer.optimizer.zero_grad() + + def before_forward(self, trainer): + pass + + def backward(self, trainer, loss_keys, cumulative_iters, grad_clip): + """Do module backward, optimizer's step and zero_grad and clip the grads. + + This is a strategic function which can be registered by other hook's function. + + Args: + trainer(`EpochBasedTrainer`): The trainer instance. + loss_keys(`list`): The list of loss keys. + cumulative_iters(`int`): The cumulative iters for gradients. + grad_clip(`dict`): The grad clipping options. + """ + for k in loss_keys: + trainer.train_outputs[k] /= cumulative_iters + trainer.train_outputs[k].backward() + + if Hook.every_n_iters(trainer, cumulative_iters): + if grad_clip is not None: + self.clip_grads(trainer.model.parameters(), **grad_clip) + + trainer.optimizer.step() + trainer.optimizer.zero_grad() + + @staticmethod + def clip_grads(params, **clip_args): + params = list( + filter(lambda p: p.requires_grad and p.grad is not None, params)) + if len(params) > 0: + return clip_grad.clip_grad_norm_(params, **clip_args) + + @HOOKS.register_module(module_name=Hooks.OptimizerHook) class OptimizerHook(Hook): """Optimizer hook @@ -36,52 +78,21 @@ class OptimizerHook(Hook): self.loss_keys = loss_keys self.cumulative_iters = cumulative_iters self.grad_clip = grad_clip + self.processor = OptimizerProcessor() - @staticmethod - def clip_grads(params, **clip_args): - params = list( - filter(lambda p: p.requires_grad and p.grad is not None, params)) - if len(params) > 0: - return clip_grad.clip_grad_norm_(params, **clip_args) - - @Hook.overload_func(name='OptimizerHook.initialize_optimizer') - def initialize_optimizer(self, trainer): - """Initialize the optimizer. - - This is a strategic function which can be registered by other hook's function. - """ - trainer.optimizer.zero_grad() + def set_processor(self, processor): + self.processor = processor def before_run(self, trainer): - self.initialize_optimizer(trainer) trainer.cumulative_iters = self.cumulative_iters + self.processor.initialize_optimizer(trainer) - @Hook.overload_func(name='OptimizerHook.backward') - def backward(self, trainer, loss_keys, cumulative_iters, grad_clip): - """Do module backward, optimizer's step and zero_grad and clip the grads. - - This is a strategic function which can be registered by other hook's function. - - Args: - trainer(`EpochBasedTrainer`): The trainer instance. - loss_keys(`list`): The list of loss keys. - cumulative_iters(`int`): The cumulative iters for gradients. - grad_clip(`dict`): The grad clipping options. - """ - for k in loss_keys: - trainer.train_outputs[k] /= cumulative_iters - trainer.train_outputs[k].backward() - - if self.every_n_iters(trainer, cumulative_iters): - if grad_clip is not None: - self.clip_grads(trainer.model.parameters(), **grad_clip) - - trainer.optimizer.step() - trainer.optimizer.zero_grad() + def before_train_iter(self, trainer): + self.processor.before_forward(trainer) def after_train_iter(self, trainer): - self.backward(trainer, self.loss_keys, self.cumulative_iters, - self.grad_clip) + self.processor.backward(trainer, self.loss_keys, self.cumulative_iters, + self.grad_clip) @HOOKS.register_module(module_name=Hooks.NoneOptimizerHook) diff --git a/modelscope/trainers/hooks/optimizer/torch_optimizer_hook.py b/modelscope/trainers/hooks/optimizer/torch_optimizer_hook.py index 1ab89720..fc7d2672 100644 --- a/modelscope/trainers/hooks/optimizer/torch_optimizer_hook.py +++ b/modelscope/trainers/hooks/optimizer/torch_optimizer_hook.py @@ -4,7 +4,45 @@ import logging from modelscope.metainfo import Hooks from modelscope.trainers.hooks import Hook from modelscope.trainers.hooks.builder import HOOKS -from .base import OptimizerHook +from .base import OptimizerHook, OptimizerProcessor + + +class TorchAMPOptimizerProcessor(OptimizerProcessor): + + def __init__(self, scaler, scale_update_param): + self.scaler = scaler + self.scale_update_param = scale_update_param + + def before_forward(self, trainer): + from torch.cuda import amp + setattr(self._model, 'forward', amp.autocast()(self._model.forward)) + + def initialize_optimizer(self, trainer): + logging.info('open fp16') + trainer.optimizer.zero_grad() + + model = trainer.unwrap_module(trainer.model) + self._ori_model_forward = model.forward + self._model = model + + def backward(self, trainer, loss_keys, cumulative_iters, grad_clip): + for k in loss_keys: + trainer.train_outputs[k] /= cumulative_iters + + for k in loss_keys: + self.scaler.scale(trainer.train_outputs[k]).backward() + + if Hook.every_n_iters(trainer, cumulative_iters): + self.scaler.unscale_(trainer.optimizer) + if grad_clip is not None: + OptimizerProcessor.clip_grads(trainer.model.parameters(), + **grad_clip) + + self.scaler.step(trainer.optimizer) + self.scaler.update(self.scale_update_param) + trainer.optimizer.zero_grad() + + setattr(self._model, 'forward', self._ori_model_forward) @HOOKS.register_module(module_name=Hooks.TorchAMPOptimizerHook) @@ -44,39 +82,11 @@ class TorchAMPOptimizerHook(Hook): '`loss_scale` type must be in [float, dict], but got {loss_scale}' ) - def register_strategy(self): - Hook.overload( - name='OptimizerHook.initialize_optimizer', - function=self.initialize_optimizer) - Hook.overload(name='OptimizerHook.backward', function=self.backward) - - def initialize_optimizer(self, trainer): - logging.info('open fp16') - trainer.optimizer.zero_grad() - - model = trainer.unwrap_module(trainer.model) - self._ori_model_forward = model.forward - self._model = model - - def before_train_iter(self, trainer): - from torch.cuda import amp - setattr(self._model, 'forward', amp.autocast()(self._model.forward)) - - def backward(self, trainer, loss_keys, cumulative_iters, grad_clip): - for k in loss_keys: - trainer.train_outputs[k] /= cumulative_iters - - for k in loss_keys: - self.scaler.scale(trainer.train_outputs[k]).backward() - - if self.every_n_iters(trainer, cumulative_iters): - self.scaler.unscale_(trainer.optimizer) - if grad_clip is not None: - OptimizerHook.clip_grads(trainer.model.parameters(), - **grad_clip) - - self.scaler.step(trainer.optimizer) - self.scaler.update(self._scale_update_param) - trainer.optimizer.zero_grad() - - setattr(self._model, 'forward', self._ori_model_forward) + def register_processor(self, trainer): + optimizer_hook = trainer.get_hook(OptimizerHook) + if len(optimizer_hook) > 0 and type( + optimizer_hook[0].processor) in (type(None), + OptimizerProcessor): + optimizer_hook[0].set_processor( + TorchAMPOptimizerProcessor(self.scaler, + self._scale_update_param)) diff --git a/modelscope/trainers/trainer.py b/modelscope/trainers/trainer.py index 683ff2f5..c980de04 100644 --- a/modelscope/trainers/trainer.py +++ b/modelscope/trainers/trainer.py @@ -11,7 +11,7 @@ import json import torch from torch import distributed as dist from torch import nn -from torch.utils.data import DataLoader, Dataset +from torch.utils.data import DataLoader, Dataset, Sampler from torch.utils.data.dataloader import default_collate from torch.utils.data.distributed import DistributedSampler @@ -88,7 +88,7 @@ class EpochBasedTrainer(BaseTrainer): compile_options (dict, optional): The compile options if compile=True, default None to use the default params of 'TorchModel.compile'. efficient_tuners (dict, optional): The tuners to use to train the model - + samplers: (:obj:`Sampler` or `Dict[Sampler]`, *optional*): samplers used in the train/eval DataLoader. Examples of cfg_modify_fn: >>> def cfg_modify_fn(cfg): >>> cfg.preprocessor.first_sequence= 'text1' @@ -114,6 +114,7 @@ class EpochBasedTrainer(BaseTrainer): model_revision: Optional[str] = DEFAULT_MODEL_REVISION, seed: int = 42, callbacks: Optional[List[Hook]] = None, + samplers: Optional[Union[Sampler, Dict[str, Sampler]]] = None, efficient_tuners: List[Dict] = None, **kwargs): @@ -132,6 +133,7 @@ class EpochBasedTrainer(BaseTrainer): self.train_dataloader = None self.eval_dataloader = None self.data_loader = None + self._samplers = samplers if isinstance(model, str): third_party = kwargs.get(ThirdParty.KEY) @@ -224,9 +226,6 @@ class EpochBasedTrainer(BaseTrainer): # Please check the DDPHook and MegatronHook for details. self.parallel_groups = {} - # Clear the Hook overload functions to avoid duplication. - Hook.clear_strategies() - if self.launcher is not None and not self.cfg.safe_get( 'train.hooks.DDPHook'): # A logic to fit the current code @@ -681,6 +680,7 @@ class EpochBasedTrainer(BaseTrainer): self.train_dataloader = self.get_train_dataloader() self.data_loader = self.train_dataloader self.register_optimizers_hook() + self.register_processors() self.print_hook_info() self.set_checkpoint_file_to_hook(checkpoint_path, load_all_state, kwargs.get('strict', False)) @@ -720,6 +720,7 @@ class EpochBasedTrainer(BaseTrainer): strict(`boolean`): If strict, any unmatched keys will cause an error. """ + self.register_processors() self.print_hook_info() if checkpoint_path is not None: from modelscope.trainers.hooks import LoadCheckpointHook @@ -758,6 +759,7 @@ class EpochBasedTrainer(BaseTrainer): kwargs: strict(`boolean`): If strict, any unmatched keys will cause an error. """ + self.register_processors() self.print_hook_info() if checkpoint_path is not None: from modelscope.trainers.hooks import LoadCheckpointHook @@ -897,11 +899,18 @@ class EpochBasedTrainer(BaseTrainer): """ if self.train_dataset is None: raise 'The train_dataset cannot be None.' + + sampler_cfg = {} + if self._samplers is not None: + sampler_cfg['sampler'] = self._samplers[ + ConfigKeys.train] if isinstance(self._samplers, + dict) else self._samplers data_loader = self._build_dataloader_with_dataset( self.train_dataset, dist=self._dist, seed=self._seed, collate_fn=self.train_data_collator, + **sampler_cfg, **self.cfg.train.get('dataloader', {})) return data_loader @@ -915,6 +924,11 @@ class EpochBasedTrainer(BaseTrainer): if self.eval_dataset is None: raise 'The eval_dataset cannot be None.' + sampler_cfg = {} + if self._samplers is not None: + sampler_cfg['sampler'] = self._samplers[ + ConfigKeys.val] if isinstance(self._samplers, + dict) else self._samplers default_config = {'shuffle': False} default_config.update(self.cfg.evaluation.get('dataloader', {})) data_loader = self._build_dataloader_with_dataset( @@ -922,6 +936,7 @@ class EpochBasedTrainer(BaseTrainer): dist=self._dist, seed=self._seed, collate_fn=self.eval_data_collator, + **sampler_cfg, **default_config) return data_loader @@ -938,6 +953,11 @@ class EpochBasedTrainer(BaseTrainer): mode=ModeKeys.EVAL, preprocessor=self.eval_preprocessor) + sampler_cfg = {} + if self._samplers is not None: + sampler_cfg['sampler'] = self._samplers[ + ConfigKeys.val] if isinstance(self._samplers, + dict) else self._samplers default_config = {'shuffle': False} default_config.update(self.cfg.evaluation.get('dataloader', {})) data_loader = self._build_dataloader_with_dataset( @@ -945,6 +965,7 @@ class EpochBasedTrainer(BaseTrainer): dist=self._dist, seed=self._seed, collate_fn=self.eval_data_collator, + **sampler_cfg, **default_config) return data_loader @@ -1132,13 +1153,19 @@ class EpochBasedTrainer(BaseTrainer): batch_size = batch_size_per_gpu num_workers = workers_per_gpu - if dist and not isinstance(dataset, torch.utils.data.IterableDataset): - sampler = DistributedSampler( - dataset, num_replicas=world_size, rank=rank, shuffle=shuffle) - else: - sampler = None - if not isinstance(dataset, torch.utils.data.IterableDataset): - kwargs['shuffle'] = shuffle + sampler = kwargs.pop('sampler', None) + if sampler is None: + if dist and not isinstance(dataset, + torch.utils.data.IterableDataset): + sampler = DistributedSampler( + dataset, + num_replicas=world_size, + rank=rank, + shuffle=shuffle) + else: + sampler = None + if not isinstance(dataset, torch.utils.data.IterableDataset): + kwargs['shuffle'] = shuffle batch_sampler = None @@ -1169,7 +1196,6 @@ class EpochBasedTrainer(BaseTrainer): """ Training loop used by `EpochBasedTrainer.train()` """ self.invoke_hook(TrainerStages.before_run) - kwargs = {} self.model.train() for _ in range(self._epoch, self._max_epochs): self.invoke_hook(TrainerStages.before_train_epoch) @@ -1181,7 +1207,7 @@ class EpochBasedTrainer(BaseTrainer): self.data_batch = data_batch self._inner_iter = i self.invoke_hook(TrainerStages.before_train_iter) - self.train_step(self.model, data_batch, **kwargs) + self.train_step(self.model, data_batch) self.invoke_hook(TrainerStages.after_train_iter) # Value changed after the hooks are invoked, do not move them above the invoke_hook code. del self.data_batch @@ -1320,12 +1346,17 @@ class EpochBasedTrainer(BaseTrainer): hooks = [] for cfg_i in hook_cfg: hook = build_from_cfg(cfg_i, HOOKS) - if hasattr(hook, 'register_strategy'): - hook.register_strategy() self.register_hook(hook) hooks.append(hook) return hooks + def register_processors(self): + """Register processors to hooks + """ + for hook in self.hooks: + if hasattr(hook, 'register_processor'): + hook.register_processor(self) + def get_hook(self, cls): return [h for h in self._hooks if h.__class__ == cls] @@ -1381,14 +1412,7 @@ class EpochBasedTrainer(BaseTrainer): info += '\n -------------------- ' stage_hook_infos.append(info) stage_hook_infos = '\n'.join(stage_hook_infos) - - strategy_info = '\n --- Hook strategies info --- \n' - for consumer, methods in Hook._strategies.items(): - strategy_info += f'Method: {consumer} ' \ - f'replaced by: ' \ - f'{[method.__self__.__class__.__name__ + "." + method.__name__ for method in methods]}\n' - strategy_info += '\n --- Hook strategies info end --- \n' - return stage_hook_infos + strategy_info + return stage_hook_infos def worker_init_fn(worker_id, num_workers, rank, seed): diff --git a/modelscope/trainers/training_args.py b/modelscope/trainers/training_args.py index a8eac217..260f2eb9 100644 --- a/modelscope/trainers/training_args.py +++ b/modelscope/trainers/training_args.py @@ -1,108 +1,544 @@ # Copyright (c) Alibaba, Inc. and its affiliates. - import re -from argparse import Action, ArgumentDefaultsHelpFormatter, ArgumentParser +from copy import deepcopy from dataclasses import dataclass, field, fields -from functools import partial -from typing import Any, Dict, List, Tuple, Union +from typing import List, Union -from modelscope.trainers.default_config import DEFAULT_CONFIG -from modelscope.utils.config import Config, ConfigDict -from modelscope.utils.hub import read_config +import addict +import json + +from modelscope.trainers.cli_argument_parser import CliArgumentParser +from modelscope.utils.config import Config -def get_flatten_value(config: Config, metadata: Dict, exclusions=None): - cfg_node = metadata['cfg_node'] - if exclusions is None: - exclusions = [] - - values = config.safe_get(cfg_node) - if isinstance(values, dict): - param_map = [] - for key, value in values.items(): - if key in exclusions or not isinstance(value, - (str, int, float, bool)): - continue - value = add_quotes_for_str(value) - param_map.append(f'{key}={value}') - return ','.join(param_map) - else: - return values - - -def set_flatten_value(config: Config, values: Union[str, List[str]], - metadata: Dict): - cfg_node = metadata['cfg_node'] - if values is None: - return config - +def set_flatten_value(values: Union[str, List[str]]): pairs = values.split(',') if isinstance(values, str) else values - for kv in pairs: + _params = {} + for kv in pairs or []: if len(kv.strip()) == 0: continue key, value = kv.split('=') - value = parse_value(value) - config.merge_from_dict({cfg_node + '.' + key: value}) - return config + _params[key] = parse_value(value) + return _params -def get_base_hook_args(config: Config, metadata: Dict): - cfg_node = metadata['cfg_node'] - hook_type = metadata['hook_type'] - key = metadata['key'] - value = config.safe_get(cfg_node) - if value is None: - return get_hook_param(config, hook_type, key) - else: - return True if key == 'type' else value +@dataclass +class DatasetArgs: + + train_dataset_name: str = field( + default=None, + metadata={ + 'help': + 'The dataset name used for training, can be an id in the datahub or a local dir', + }) + + val_dataset_name: str = field( + default=None, + metadata={ + 'help': + 'The subset name used for evaluating, can be an id in the datahub or a local dir', + }) + + train_subset_name: str = field( + default=None, + metadata={ + 'help': 'The subset name used for training, can be None', + }) + + val_subset_name: str = field( + default=None, + metadata={ + 'help': 'The subset name used for evaluating, can be None', + }) + + train_dataset_namespace: str = field( + default=None, + metadata={ + 'help': 'The dataset namespace used for training', + }) + + val_dataset_namespace: str = field( + default=None, + metadata={ + 'help': 'The dataset namespace used for evaluating', + }) + + dataset_json_file: str = field( + default=None, + metadata={ + 'help': + 'The json file to parse all datasets from, used in a complex dataset scenario,' + 'the json format should be like:' + ''' + [ + { + "dataset": { + # All args used in the MsDataset.load function + "dataset_name": "xxx", + ... + }, + # All columns used, mapping the column names in each dataset in same names. + "column_mapping": { + "text1": "sequence1", + "text2": "sequence2", + "label": "label", + }, + # float or str, float means to split the dataset into train/val, + # or just str(train/val) + "split": 0.8, + } + ] + ''', + }) -def set_base_hook_args(config: Config, value: Any, metadata: Dict): - cfg_node = metadata['cfg_node'] - hook_type = metadata['hook_type'] - key = metadata['key'] - if 'hooks' in config.train: - config.train.hooks = [ - hook for hook in config.train.hooks if hook['type'] != hook_type - ] - if key == 'type': - if value and config.safe_get(cfg_node) is None: - config.merge_from_dict({cfg_node: {}}) - else: - config.merge_from_dict({cfg_node: value}) +@dataclass +class ModelArgs: + task: str = field( + default=None, + metadata={ + 'help': 'The task code to be used', + 'cfg_node': 'task' + }) + + model: str = field( + default=None, metadata={ + 'help': 'A model id or model dir', + }) + + model_type: str = field( + default=None, + metadata={ + 'help': + 'The mode type, if load_model_config is False, user need to fill this field', + 'cfg_node': 'model.type' + }) -def get_strategy(config: Config, - metadata: Dict, - value_pair: Tuple[str] = ('by_epoch', 'by_step')): - flag = get_base_hook_args(config, metadata) - if flag is None: +@dataclass +class TrainArgs: + + seed: int = field( + default=42, metadata={ + 'help': 'The random seed', + }) + + per_device_train_batch_size: int = field( + default=16, + metadata={ + 'cfg_node': 'train.dataloader.batch_size_per_gpu', + 'help': + 'The `batch_size_per_gpu` argument for the train dataloader', + }) + + train_data_worker: int = field( + default=0, + metadata={ + 'cfg_node': 'train.dataloader.workers_per_gpu', + 'help': 'The `workers_per_gpu` argument for the train dataloader', + }) + + train_shuffle: bool = field( + default=False, + metadata={ + 'cfg_node': 'train.dataloader.shuffle', + 'help': 'The `shuffle` argument for the train dataloader', + }) + + train_drop_last: bool = field( + default=False, + metadata={ + 'cfg_node': 'train.dataloader.drop_last', + 'help': 'The `drop_last` argument for the train dataloader', + }) + + per_device_eval_batch_size: int = field( + default=16, + metadata={ + 'cfg_node': 'evaluation.dataloader.batch_size_per_gpu', + 'help': + 'The `batch_size_per_gpu` argument for the eval dataloader', + }) + + eval_data_worker: int = field( + default=0, + metadata={ + 'cfg_node': 'evaluation.dataloader.workers_per_gpu', + 'help': 'The `workers_per_gpu` argument for the eval dataloader', + }) + + eval_shuffle: bool = field( + default=False, + metadata={ + 'cfg_node': 'evaluation.dataloader.shuffle', + 'help': 'The `shuffle` argument for the eval dataloader', + }) + + eval_drop_last: bool = field( + default=False, + metadata={ + 'cfg_node': 'evaluation.dataloader.drop_last', + 'help': 'The `drop_last` argument for the eval dataloader', + }) + + max_epochs: int = field( + default=5, + metadata={ + 'cfg_node': 'train.max_epochs', + 'help': 'The training epochs', + }) + + work_dir: str = field( + default='./train_target', + metadata={ + 'cfg_node': 'train.work_dir', + 'help': 'The directory to save models and logs', + }) + + lr: float = field( + default=5e-5, + metadata={ + 'cfg_node': 'train.optimizer.lr', + 'help': 'The learning rate of the optimizer', + }) + + lr_scheduler: str = field( + default='LinearLR', + metadata={ + 'cfg_node': 'train.lr_scheduler.type', + 'help': 'The lr_scheduler type in torch', + }) + + optimizer: str = field( + default='AdamW', + metadata={ + 'cfg_node': 'train.optimizer.type', + 'help': 'The optimizer type in PyTorch, like `AdamW`', + }) + + optimizer_params: str = field( + default=None, + metadata={ + 'cfg_node': 'train.optimizer', + 'help': 'The optimizer params', + 'cfg_setter': set_flatten_value, + }) + + lr_scheduler_params: str = field( + default=None, + metadata={ + 'cfg_node': 'train.lr_scheduler', + 'help': 'The lr scheduler params', + 'cfg_setter': set_flatten_value, + }) + + lr_strategy: str = field( + default='by_epoch', + metadata={ + 'cfg_node': 'train.lr_scheduler.options.lr_strategy', + 'help': 'The lr decay strategy', + 'choices': ['by_epoch', 'by_step', 'no'], + }) + + local_rank: int = field( + default=0, metadata={ + 'help': 'The local rank', + }) + + logging_interval: int = field( + default=5, + metadata={ + 'help': 'The interval of iter of logging information', + 'cfg_node': 'train.logging.interval', + }) + + eval_strategy: str = field( + default='by_epoch', + metadata={ + 'help': 'Eval strategy, can be `by_epoch` or `by_step` or `no`', + 'cfg_node': 'evaluation.period.eval_strategy', + 'choices': ['by_epoch', 'by_step', 'no'], + }) + + eval_interval: int = field( + default=1, + metadata={ + 'help': 'Eval interval', + 'cfg_node': 'evaluation.period.interval', + }) + + eval_metrics: str = field( + default=None, + metadata={ + 'help': 'The metric name for evaluation', + 'cfg_node': 'evaluation.metrics' + }) + + save_strategy: str = field( + default='by_epoch', + metadata={ + 'help': + 'Checkpointing strategy, can be `by_epoch` or `by_step` or `no`', + 'cfg_node': 'train.checkpoint.period.save_strategy', + 'choices': ['by_epoch', 'by_step', 'no'], + }) + + save_interval: int = field( + default=1, + metadata={ + 'help': + 'The interval of epoch or iter of saving checkpoint period', + 'cfg_node': 'train.checkpoint.period.interval', + }) + + save_best_checkpoint: bool = field( + default=False, + metadata={ + 'help': + 'Save the checkpoint(if it\'s the best) after the evaluation.', + 'cfg_node': 'train.checkpoint.best.save_best', + }) + + metric_for_best_model: str = field( + default=None, + metadata={ + 'help': 'The metric used to measure the model.', + 'cfg_node': 'train.checkpoint.best.metric_key', + }) + + metric_rule_for_best_model: str = field( + default='max', + metadata={ + 'help': + 'The rule to measure the model with the metric, can be `max` or `min`', + 'cfg_node': 'train.checkpoint.best.rule', + }) + + max_checkpoint_num: int = field( + default=None, + metadata={ + 'help': + 'The max number of checkpoints to keep, older ones will be deleted.', + 'cfg_node': 'train.checkpoint.period.max_checkpoint_num', + }) + + max_checkpoint_num_best: int = field( + default=1, + metadata={ + 'help': + 'The max number of best checkpoints to keep, worse ones will be deleted.', + 'cfg_node': 'train.checkpoint.best.max_checkpoint_num', + }) + + push_to_hub: bool = field( + default=False, + metadata={ + 'help': 'Push to hub after each checkpointing', + 'cfg_node': 'train.checkpoint.period.push_to_hub', + }) + + repo_id: str = field( + default=None, + metadata={ + 'help': + 'The repo id in modelhub, usually the format is "group/model"', + 'cfg_node': 'train.checkpoint.period.hub_repo_id', + }) + + hub_token: str = field( + default=None, + metadata={ + 'help': + 'The modelhub token, you can also set the token to the env variable `MODELSCOPE_API_TOKEN`', + 'cfg_node': 'train.checkpoint.period.hub_token', + }) + + private_hub: bool = field( + default=True, + metadata={ + 'help': 'Upload to a private hub', + 'cfg_node': 'train.checkpoint.period.private_hub', + }) + + hub_revision: str = field( + default='master', + metadata={ + 'help': 'Which branch to commit to', + 'cfg_node': 'train.checkpoint.period.hub_revision', + }) + + push_to_hub_best: bool = field( + default=False, + metadata={ + 'help': 'Push to hub after each checkpointing', + 'cfg_node': 'train.checkpoint.best.push_to_hub', + }) + + repo_id_best: str = field( + default=None, + metadata={ + 'help': + 'The repo id in modelhub, usually the format is "group/model"', + 'cfg_node': 'train.checkpoint.best.hub_repo_id', + }) + + hub_token_best: str = field( + default=None, + metadata={ + 'help': + 'The modelhub token, you can also set the token to the env variable `MODELSCOPE_API_TOKEN`', + 'cfg_node': 'train.checkpoint.best.hub_token', + }) + + private_hub_best: bool = field( + default=True, + metadata={ + 'help': 'Upload to a private hub', + 'cfg_node': 'train.checkpoint.best.private_hub', + }) + + hub_revision_best: str = field( + default='master', + metadata={ + 'help': 'Which branch to commit to', + 'cfg_node': 'train.checkpoint.best.hub_revision', + }) + + +@dataclass(init=False) +class TrainingArgs(DatasetArgs, TrainArgs, ModelArgs): + + use_model_config: bool = field( + default=False, + metadata={ + 'help': + 'Use the configuration of the model, ' + 'default will only use the parameters in the CLI and the dataclass', + }) + + def __init__(self, **kwargs): + self.manual_args = list(kwargs.keys()) + for f in fields(self): + if f.name in kwargs: + setattr(self, f.name, kwargs[f.name]) + self._unknown_args = {} + + def parse_cli(self, parser_args=None): + """Construct a TrainingArg class by the parameters of CLI. + + Returns: + Self + """ + parser = CliArgumentParser(self) + args, unknown = parser.parse_known_args(parser_args) + unknown = [item for item in unknown if item not in ('\\', '\n')] + _unknown = {} + for i in range(0, len(unknown), 2): + _unknown[unknown[i].replace('-', '')] = parse_value(unknown[i + 1]) + args_dict = vars(args) + self.manual_args += parser.manual_args + + for key, value in deepcopy(args_dict).items(): + if key is not None and hasattr(self, key): + setattr(self, key, value) + return self + + def to_config(self, ignore_default_config=None): + """Convert the TrainingArgs to the `Config` + + Returns: + The Config, and extra parameters in dict. + """ + cfg = Config() + args_dict = addict.Dict() + + if ignore_default_config is None: + ignore_default_config = self.use_model_config + + for f in fields(self): + cfg_node = f.metadata.get('cfg_node') + cfg_setter = f.metadata.get('cfg_setter') or (lambda x: x) + if cfg_node is not None: + if f.name in self.manual_args or not ignore_default_config: + cfg.merge_from_dict( + {cfg_node: cfg_setter(getattr(self, f.name))}) + else: + args_dict[f.name] = getattr(self, f.name) + + cfg.merge_from_dict(self._unknown_args) + return cfg, args_dict + + def get_metadata(self, key): + _fields = fields(self) + for f in _fields: + if f.name == key: + return f return None - return value_pair[0] if flag else value_pair[1] -def set_strategy(config: Config, - value: Any, - metadata: Dict, - value_pair: Tuple[str] = ('by_epoch', 'by_step')): - set_base_hook_args(config, value == value_pair[0], metadata) +def build_dataset_from_file(filename): + """ + The filename format: + [ + { + "dataset": { + "dataset_name": "xxx", + ... + }, + "column_mapping": { + "text1": "sequence1", + "text2": "sequence2", + "label": "label", + } + "split": 0.8, + } + ] + """ + from modelscope import MsDataset + train_set = [] + eval_set = [] + with open(filename, 'r') as f: + ds_json = json.load(f) + for ds in ds_json: + dataset = MsDataset.load(**ds['dataset']).to_hf_dataset() + all_columns = dataset.column_names + keep_columns = ds['column_mapping'].keys() + remove_columns = [ + column for column in all_columns if column not in keep_columns + ] + from datasets import Features + from datasets import Value + from datasets import ClassLabel + features = [ + f for f in dataset.features.items() if f[0] in keep_columns + ] + new_features = {} + for f in features: + if isinstance(f[1], ClassLabel): + new_features[f[0]] = Value(f[1].dtype) + else: + new_features[f[0]] = f[1] + new_features = Features(new_features) + dataset = dataset.map( + lambda x: x, + remove_columns=remove_columns, + features=new_features).rename_columns(ds['column_mapping']) + split = ds['split'] + if isinstance(split, str): + assert split in ('train', 'val') + if split == 'train': + train_set.append(dataset) + else: + eval_set.append(dataset) + else: + assert isinstance(split, float) and 0 < split < 1 + ds_dict = dataset.train_test_split(train_size=split) + train_set.append(ds_dict['train']) + eval_set.append(ds_dict['test']) -def get_hook_param(config, hook_type: str, key='type'): - hooks = config.safe_get('train.hooks', []) - _hooks = list(filter(lambda hook: hook['type'] == hook_type, hooks)) - if key == 'type': - return len(_hooks) > 0 - elif len(_hooks) > 0: - return getattr(_hooks[0], key, None) - return None - - -def add_quotes_for_str(value: Union[str, float, bool, None]) -> str: - if isinstance(value, str): - return f'"{value}"' - else: - return str(value) + from datasets import concatenate_datasets + return concatenate_datasets(train_set), concatenate_datasets(eval_set) def parse_value(value: str) -> Union[str, float, bool, None]: @@ -126,714 +562,3 @@ def parse_value(value: str) -> Union[str, float, bool, None]: return float(value) else: return value - - -@dataclass -class TrainingArgs: - model: str = field( - default=None, metadata={ - 'help': 'A model id or model dir', - }) - - seed: int = field( - default=42, metadata={ - 'help': 'The random seed', - }) - - task: str = field( - default=None, - metadata={ - 'help': 'The task code to be used', - 'cfg_node': 'task' - }) - - dataset_name: str = field( - default=None, metadata={ - 'help': 'The dataset name', - }) - - subset_name: str = field( - default=None, metadata={ - 'help': 'The subset name of the dataset', - }) - - train_dataset_name: str = field( - default=None, metadata={ - 'help': 'The train dataset name', - }) - - val_dataset_name: str = field( - default=None, metadata={ - 'help': 'The validation dataset name', - }) - - per_device_train_batch_size: int = field( - default=None, - metadata={ - 'cfg_node': 'train.dataloader.batch_size_per_gpu', - 'help': 'The training batch size per GPU', - }) - - train_data_worker: int = field( - default=0, - metadata={ - 'cfg_node': 'train.dataloader.workers_per_gpu', - 'help': 'The number of data workers for train dataloader', - }) - - train_shuffle: bool = field( - default=None, - metadata={ - 'cfg_node': 'train.dataloader.shuffle', - 'help': 'Shuffle the train dataset or not', - }) - - train_drop_last: bool = field( - default=None, - metadata={ - 'cfg_node': - 'train.dataloader.drop_last', - 'help': - 'Whether to drop out the last set of data in the train_dataset', - }) - - per_device_eval_batch_size: int = field( - default=None, - metadata={ - 'cfg_node': 'evaluation.dataloader.batch_size_per_gpu', - 'help': 'The eval batch size per GPU', - }) - - eval_data_worker: int = field( - default=0, - metadata={ - 'cfg_node': 'evaluation.dataloader.workers_per_gpu', - 'help': 'The number of data workers for eval dataloader', - }) - - eval_shuffle: bool = field( - default=None, - metadata={ - 'cfg_node': 'evaluation.dataloader.shuffle', - 'help': 'Shuffle the eval dataset or not', - }) - - eval_drop_last: bool = field( - default=None, - metadata={ - 'cfg_node': 'evaluation.dataloader.drop_last', - 'help': - 'Whether to drop out the last set of data in the eval_dataset', - }) - - max_epochs: int = field( - default=None, - metadata={ - 'cfg_node': 'train.max_epochs', - 'help': 'The training epochs', - }) - - work_dir: str = field( - default=None, - metadata={ - 'cfg_node': 'train.work_dir', - 'help': 'The training dir to save models and logs', - }) - - lr: float = field( - default=None, - metadata={ - 'cfg_node': 'train.optimizer.lr', - 'help': 'The learning rate of the optimizer', - }) - - optimizer: str = field( - default=None, - metadata={ - 'cfg_node': 'train.optimizer.type', - 'help': 'The optimizer type', - }) - - optimizer_params: str = field( - default=None, - metadata={ - 'cfg_node': - 'train.optimizer', - 'cfg_getter': - partial(get_flatten_value, exclusions=['type', 'lr', 'options']), - 'cfg_setter': - set_flatten_value, - 'help': - 'The optimizer init params except `lr`', - }) - - lr_scheduler_params: str = field( - default=None, - metadata={ - 'cfg_node': - 'train.lr_scheduler', - 'cfg_getter': - partial(get_flatten_value, exclusions=['type', 'lr', 'options']), - 'cfg_setter': - set_flatten_value, - 'help': - 'The lr_scheduler init params', - }) - - local_rank: int = field( - default=0, metadata={ - 'help': 'The training local rank', - }) - - save_ckpt: bool = field( - default=True, - metadata={ - 'help': - 'Periodically save checkpoint when True, corresponding to CheckpointHook', - 'cfg_node': 'train.checkpoint.period', - 'hook_type': 'CheckpointHook', - 'key': 'type', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - save_ckpt_best: bool = field( - default=None, - metadata={ - 'help': - 'Save best checkpoint when True, corresponding to BestCkptSaverHook', - 'cfg_node': 'train.checkpoint.best', - 'hook_type': 'BestCkptSaverHook', - 'key': 'type', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - evaluate: bool = field( - default=True, - metadata={ - 'help': 'Evaluate when True, corresponding to EvaluationHook', - 'cfg_node': 'evaluation.period', - 'hook_type': 'EvaluationHook', - 'key': 'type', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - save_ckpt_strategy: str = field( - default=None, - metadata={ - 'help': 'Periodically save checkpoint by epoch or by step' - 'use with `CheckpointHook`, can be `by_epoch` or `by_step`', - 'cfg_node': 'train.checkpoint.period.by_epoch', - 'hook_type': 'CheckpointHook', - 'key': 'by_epoch', - 'choices': ['by_epoch', 'by_step'], - 'cfg_getter': get_strategy, - 'cfg_setter': set_strategy, - }) - - save_ckpt_best_strategy: str = field( - default=None, - metadata={ - 'help': 'Save best checkpoint by epoch or by step' - 'use with `BestCkptSaverHook`, can be `by_epoch` or `by_step`', - 'cfg_node': 'train.checkpoint.best.by_epoch', - 'hook_type': 'BestCkptSaverHook', - 'key': 'by_epoch', - 'choices': ['by_epoch', 'by_step'], - 'cfg_getter': get_strategy, - 'cfg_setter': set_strategy, - }) - - push_to_hub: bool = field( - default=None, - metadata={ - 'help': - 'Push to hub after one checkpoint saved by CheckpointHook in the local disk', - 'cfg_node': 'train.checkpoint.period.push_to_hub', - 'hook_type': 'CheckpointHook', - 'key': 'push_to_hub', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - model_id_with_org: str = field( - default=None, - metadata={ - 'help': - 'The repo id in modelhub, usually it\'s like "group/model"', - 'cfg_node': 'train.checkpoint.period.model_id_with_org', - 'hook_type': 'CheckpointHook', - 'key': 'model_id_with_org', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - hub_token: str = field( - default=None, - metadata={ - 'help': - 'The token to push to hub, you can also set the token to the env variable `MODELSCOPE_API_TOKEN`', - 'cfg_node': 'train.checkpoint.period.hub_token', - 'hook_type': 'CheckpointHook', - 'key': 'hub_token', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - private_hub: bool = field( - default=None, - metadata={ - 'help': 'Upload to a private hub', - 'cfg_node': 'train.checkpoint.period.private_hub', - 'hook_type': 'CheckpointHook', - 'key': 'private_hub', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - push_to_hub_best_model: bool = field( - default=None, - metadata={ - 'help': - 'Push to hub after one checkpoint saved by BestCkptSaverHook in the local disk', - 'cfg_node': 'train.checkpoint.best.push_to_hub', - 'hook_type': 'BestCkptSaverHook', - 'key': 'push_to_hub', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - model_id_with_org_best_model: str = field( - default=None, - metadata={ - 'help': - 'The repo id in modelhub, usually it\'s like "group/model"', - 'cfg_node': 'train.checkpoint.best.model_id_with_org', - 'hook_type': 'BestCkptSaverHook', - 'key': 'model_id_with_org', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - hub_token_best_model: str = field( - default=None, - metadata={ - 'help': - 'The token to push to hub, you can also set the token to the env variable `MODELSCOPE_API_TOKEN`', - 'cfg_node': 'train.checkpoint.best.hub_token', - 'hook_type': 'BestCkptSaverHook', - 'key': 'hub_token', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - private_hub_best_model: bool = field( - default=None, - metadata={ - 'help': 'Upload to a private hub', - 'cfg_node': 'train.checkpoint.best.private_hub', - 'hook_type': 'BestCkptSaverHook', - 'key': 'private_hub', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - ckpt_period_interval: int = field( - default=1, - metadata={ - 'help': - 'The interval of epoch or iter of saving checkpoint period', - 'cfg_node': 'train.checkpoint.period.interval', - 'hook_type': 'CheckpointHook', - 'key': 'interval', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - ckpt_best_interval: int = field( - default=None, - metadata={ - 'help': 'The interval of epoch or iter of saving checkpoint best', - 'cfg_node': 'train.checkpoint.best.interval', - 'hook_type': 'BestCkptSaverHook', - 'key': 'interval', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - metric_for_best_model: str = field( - default=None, - metadata={ - 'help': - 'Which metric key to judge the checkpoint is better or not, use with `BestCkptSaverHook`, ' - 'please make sure this key is returned by the `evaluation_metrics` classes', - 'cfg_node': - 'train.checkpoint.best.metric_key', - 'hook_type': - 'BestCkptSaverHook', - 'key': - 'metric_key', - 'cfg_getter': - get_base_hook_args, - 'cfg_setter': - set_base_hook_args, - }) - - metric_rule_for_best_model: str = field( - default=None, - metadata={ - 'help': - 'Which rule to compare the value of `checkpoint_saving_metric`, ' - 'use with `BestCkptSaverHook`, can be `max` or `min`', - 'cfg_node': - 'train.checkpoint.best.rule', - 'hook_type': - 'BestCkptSaverHook', - 'key': - 'rule', - 'cfg_getter': - get_base_hook_args, - 'cfg_setter': - set_base_hook_args, - }) - - save_ckpt_peroid_limit: int = field( - default=None, - metadata={ - 'help': - 'The max saving number of checkpoint, older checkpoints will be deleted.', - 'cfg_node': 'train.checkpoint.period.max_checkpoint_num', - 'hook_type': 'CheckpointHook', - 'key': 'max_checkpoint_num', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - save_ckpt_best_limit: int = field( - default=None, - metadata={ - 'help': - 'The max saving number of checkpoint, worse checkpoints will be deleted.', - 'cfg_node': 'train.checkpoint.best.max_checkpoint_num', - 'hook_type': 'BestCkptSaverHook', - 'key': 'max_checkpoint_num', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - logging_interval: int = field( - default=None, - metadata={ - 'help': 'The interval of iter of logging information', - 'cfg_node': 'train.logging.interval', - 'hook_type': 'TextLoggerHook', - 'key': 'interval', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - eval_strategy: str = field( - default=None, - metadata={ - 'help': 'Evaluate model by epoch or by step' - 'use with `EvaluationHook`, can be `by_epoch` or `by_step`', - 'cfg_node': 'evaluation.period.by_epoch', - 'hook_type': 'EvaluationHook', - 'key': 'by_epoch', - 'choices': ['by_epoch', 'by_step'], - 'cfg_getter': get_strategy, - 'cfg_setter': set_strategy, - }) - - eval_interval: int = field( - default=None, - metadata={ - 'help': 'Evaluation interval by epoch or iter', - 'cfg_node': 'evaluation.period.interval', - 'hook_type': 'EvaluationHook', - 'key': 'interval', - 'cfg_getter': get_base_hook_args, - 'cfg_setter': set_base_hook_args, - }) - - eval_metrics: str = field( - default=None, - metadata={ - 'help': 'The metric module name used in evaluation', - 'cfg_node': 'evaluation.metrics' - }) - - @classmethod - def from_cli(cls, parser_args=None, **extra_kwargs): - """Construct a TrainingArg class by the parameters of CLI. - - Args: - **extra_kwargs: Extra args which can be defined in code. - - Returns: - The output TrainingArg class with the parameters from CLI. - """ - self = cls(**extra_kwargs) - parser = CliArgumentParser(self) - args, unknown = parser.parse_known_args(parser_args) - unknown = [item for item in unknown if item not in ('\\', '\n')] - _unknown = {} - for i in range(0, len(unknown), 2): - _unknown[unknown[i].replace('-', '')] = parse_value(unknown[i + 1]) - cfg_dict = vars(args) - - if args.model is not None: - try: - cfg = read_config(args.model) - except Exception as e: - print('Read config failed with error:', e) - else: - self = cls.from_config(cfg, **extra_kwargs) - for key, value in cfg_dict.items(): - if key is not None and hasattr(self, - key) and key in parser.manual_args: - setattr(self, key, value) - self.extra_args = _unknown - return self - - def to_args(self): - """Convert the TrainingArg class to key-value pairs. - - Returns: The key-value pair. - - """ - _args = {} - for f in fields(self): - _args[f.name] = getattr(self, f.name) - return _args - - @classmethod - def from_config(cls, config=DEFAULT_CONFIG, **kwargs): - """Construct the TrainingArg class by a `Config` class. - - Args: - config: The Config class. By default, `DEFAULT_CONFIG` is used. - **kwargs: Extra args which can be defined in code. - - Returns: The output TrainingArg class with the parameters from the config. - - """ - - self = cls(**kwargs) - for f in fields(self): - if 'cfg_node' in f.metadata and getattr(self, f.name) is None: - self._to_field(f, config) - return self - - def _to_field(self, f, config): - assert 'cfg_node' in f.metadata - if 'cfg_getter' in f.metadata: - cfg_getter = f.metadata['cfg_getter'] - setattr(self, f.name, cfg_getter(config, f.metadata)) - else: - cfg_node = f.metadata['cfg_node'] - setattr(self, f.name, config.safe_get(cfg_node)) - - def _to_config(self, f, config: Config): - assert 'cfg_node' in f.metadata - value = getattr(self, f.name) - if 'cfg_setter' in f.metadata: - cfg_setter = f.metadata['cfg_setter'] - config = cfg_setter(config, value, f.metadata) - else: - cfg_node = f.metadata['cfg_node'] - if isinstance(cfg_node, str): - cfg_node = [cfg_node] - for _node in cfg_node: - config.merge_from_dict({_node: value}) - return config - - def __call__(self, cfg: Config): - for f in fields(self): - if 'cfg_node' not in f.metadata: - continue - - value = getattr(self, f.name) - if value is not None: - self._to_config(f, cfg) - if hasattr(self, 'extra_args'): - cfg.merge_from_dict(self.extra_args) - else: - self._to_field(f, cfg) - return cfg - - -class CliArgumentParser(ArgumentParser): - """ Argument Parser to define and parse command-line args for training. - - Args: - training_args (TrainingArgs): dict or list of dict which defines different - paramters for training. - """ - - def __init__(self, training_args: TrainingArgs = None, **kwargs): - if 'formatter_class' not in kwargs: - kwargs['formatter_class'] = ArgumentDefaultsHelpFormatter - super().__init__(**kwargs) - self.training_args = training_args - self.define_args() - - def get_manual_args(self, args): - return [arg[2:] for arg in args if arg.startswith('--')] - - def _parse_known_args(self, args: List = None, namespace=None): - self.model_id = namespace.model if namespace is not None else None - if '--model' in args: - self.model_id = args[args.index('--model') + 1] - self.manual_args = self.get_manual_args(args) - return super()._parse_known_args(args, namespace) - - def print_help(self, file=None): - config = DEFAULT_CONFIG - if self.model_id is not None: - try: - config = read_config(self.model_id) - except Exception as e: - print('Read config failed with error:', e) - - if config is not None: - for action_group in self._optionals._group_actions: - if hasattr(self.training_args, action_group.dest): - value = getattr(self.training_args, action_group.dest) - f = {f.name: f - for f in fields(self.training_args) - }.get(action_group.dest) - if value is not None: - action_group.default = value - elif 'cfg_node' in f.metadata: - cfg_node = f.metadata['cfg_node'] - if isinstance(cfg_node, str): - cfg_node = [cfg_node] - - assert isinstance(cfg_node, (list, tuple)) - if isinstance(cfg_node[0], str): - action_group.default = config.safe_get(cfg_node[0]) - else: - action_group.default = cfg_node[0](config) - return super().print_help(file) - - def define_args(self): - if self.training_args is not None: - for f in fields(self.training_args): - arg_name = f.name - arg_attr = getattr(self.training_args, f.name) - name = f'--{arg_name}' - kwargs = dict(type=f.type, help=f.metadata['help']) - kwargs['default'] = arg_attr - - if 'choices' in f.metadata: - kwargs['choices'] = f.metadata['choices'] - - kwargs['action'] = SingleAction - self.add_argument(name, **kwargs) - - -class DictAction(Action): - """ - argparse action to split an argument into KEY=VALUE form - on the first = and append to a dictionary. List options can - be passed as comma separated values, i.e 'KEY=V1,V2,V3', or with explicit - brackets, i.e. 'KEY=[V1,V2,V3]'. It also support nested brackets to build - list/tuple values. e.g. 'KEY=[(V1,V2),(V3,V4)]' - """ - - @staticmethod - def parse_int_float_bool_str(val): - try: - return int(val) - except ValueError: - pass - try: - return float(val) - except ValueError: - pass - if val.lower() in ['true', 'false']: - return val.lower() == 'true' - if val == 'None': - return None - return val - - @staticmethod - def parse_iterable(val): - """Parse iterable values in the string. - All elements inside '()' or '[]' are treated as iterable values. - Args: - val (str): Value string. - Returns: - list | tuple: The expanded list or tuple from the string. - Examples: - >>> DictAction._parse_iterable('1,2,3') - [1, 2, 3] - >>> DictAction._parse_iterable('[a, b, c]') - ['a', 'b', 'c'] - >>> DictAction._parse_iterable('[(1, 2, 3), [a, b], c]') - [(1, 2, 3), ['a', 'b'], 'c'] - """ - - def find_next_comma(string): - """Find the position of next comma in the string. - If no ',' is found in the string, return the string length. All - chars inside '()' and '[]' are treated as one element and thus ',' - inside these brackets are ignored. - """ - assert (string.count('(') == string.count(')')) and ( - string.count('[') - == string.count(']')), f'Imbalanced brackets exist in {string}' - end = len(string) - for idx, char in enumerate(string): - pre = string[:idx] - # The string before this ',' is balanced - if ((char == ',') and (pre.count('(') == pre.count(')')) - and (pre.count('[') == pre.count(']'))): - end = idx - break - return end - - # Strip ' and " characters and replace whitespace. - val = val.strip('\'\"').replace(' ', '') - is_tuple = False - if val.startswith('(') and val.endswith(')'): - is_tuple = True - val = val[1:-1] - elif val.startswith('[') and val.endswith(']'): - val = val[1:-1] - elif ',' not in val: - # val is a single value - return DictAction.parse_int_float_bool_str(val) - - values = [] - while len(val) > 0: - comma_idx = find_next_comma(val) - element = DictAction.parse_iterable(val[:comma_idx]) - values.append(element) - val = val[comma_idx + 1:] - if is_tuple: - values = tuple(values) - return values - - def __call__(self, parser, namespace, values, option_string): - options = {} - for kv in values: - key, val = kv.split('=', maxsplit=1) - options[key] = self.parse_iterable(val) - setattr(namespace, self.dest, options) - - -class SingleAction(DictAction): - """ Argparse action to convert value to tuple or list or nested structure of - list and tuple, i.e 'V1,V2,V3', or with explicit brackets, i.e. '[V1,V2,V3]'. - It also support nested brackets to build list/tuple values. e.g. '[(V1,V2),(V3,V4)]' - """ - - def __call__(self, parser, namespace, value, option_string): - if isinstance(value, str): - setattr(namespace, self.dest, self.parse_iterable(value)) - else: - setattr(namespace, self.dest, value) diff --git a/modelscope/utils/ast_utils.py b/modelscope/utils/ast_utils.py index 374ada20..5cee374d 100644 --- a/modelscope/utils/ast_utils.py +++ b/modelscope/utils/ast_utils.py @@ -1,7 +1,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import ast -import contextlib import hashlib import os import os.path as osp @@ -9,12 +8,11 @@ import time import traceback from functools import reduce from pathlib import Path -from typing import Generator, Union +from typing import Union import gast import json -from modelscope import __version__ from modelscope.fileio.file import LocalStorage from modelscope.metainfo import (CustomDatasets, Heads, Hooks, LR_Schedulers, Metrics, Models, Optimizers, Pipelines, @@ -574,6 +572,7 @@ file_scanner = FilesAstScanning() def _save_index(index, file_path, file_list=None, with_template=False): # convert tuple key to str key index[INDEX_KEY] = {str(k): v for k, v in index[INDEX_KEY].items()} + from modelscope.version import __version__ index[VERSION_KEY] = __version__ index[MD5_KEY], index[FILES_MTIME_KEY] = file_scanner.files_mtime_md5( file_list=file_list) @@ -682,6 +681,7 @@ def load_index( if not force_rebuild and os.path.exists(file_path): wrapped_index = _load_index(file_path) md5, files_mtime = file_scanner.files_mtime_md5(file_list=file_list) + from modelscope.version import __version__ if (wrapped_index[VERSION_KEY] == __version__): index = wrapped_index if (wrapped_index[MD5_KEY] != md5): diff --git a/modelscope/utils/checkpoint.py b/modelscope/utils/checkpoint.py index 64681db4..bbde6034 100644 --- a/modelscope/utils/checkpoint.py +++ b/modelscope/utils/checkpoint.py @@ -5,7 +5,6 @@ import os import re import time from collections import OrderedDict -from functools import partial from shutil import copytree, ignore_patterns, rmtree from typing import Callable, Dict, Optional, Union @@ -15,7 +14,6 @@ from torch import nn from torch.optim import Optimizer from torch.optim.lr_scheduler import _LRScheduler -from modelscope import __version__ from modelscope.fileio import File, LocalStorage from modelscope.utils.config import Config, JSONIteratorEncoder from modelscope.utils.constant import ConfigFields, ModelFile @@ -76,6 +74,7 @@ def save_checkpoint(model: torch.nn.Module, elif not isinstance(meta, dict): raise TypeError( f'meta must be a dict or None, but got {type(meta)}') + from modelscope import __version__ meta.update(modelscope=__version__, time=time.asctime()) if isinstance(model, torch.nn.parallel.DistributedDataParallel): diff --git a/tests/hub/test_hub_upload.py b/tests/hub/test_hub_upload.py index a8b90288..2a66cb8b 100644 --- a/tests/hub/test_hub_upload.py +++ b/tests/hub/test_hub_upload.py @@ -37,6 +37,7 @@ class HubUploadTest(unittest.TestCase): os.mkdir(self.finetune_path) os.system("echo '{}'>%s" % os.path.join(self.finetune_path, ModelFile.CONFIGURATION)) + os.environ['MODELSCOPE_TRAIN_ID'] = 'test-id' def tearDown(self): logger.info('TearDown') diff --git a/tests/trainers/cli/__init__.py b/tests/trainers/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/trainers/cli/test_cli.py b/tests/trainers/cli/test_cli.py new file mode 100644 index 00000000..b9fb7539 --- /dev/null +++ b/tests/trainers/cli/test_cli.py @@ -0,0 +1,52 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import unittest + +import json + +from modelscope import MsDataset, TrainingArgs, build_dataset_from_file +from modelscope.utils.test_utils import test_level + + +class TestCli(unittest.TestCase): + + def setUp(self) -> None: + content = [{ + 'dataset': { + 'dataset_name': 'clue', + 'subset_name': 'cmnli', + 'split': 'train', + }, + 'column_mapping': { + 'sentence1': 'sentence1', + 'sentence2': 'sentence2', + 'label': 'label', + }, + 'split': 0.8, + }, { + 'dataset': { + 'dataset_name': 'glue', + 'subset_name': 'mnli', + 'split': 'validation_matched', + }, + 'column_mapping': { + 'premise': 'sentence1', + 'hypothesis': 'sentence2', + 'label': 'label', + }, + 'split': 'val', + }] + with open('./dataset.json', 'w') as f: + json.dump(content, f) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_merge_dataset_from_file(self): + dataset = MsDataset.load('clue', subset_name='cmnli', split='train') + dataset2 = MsDataset.load( + 'glue', subset_name='mnli', split='validation_matched') + training_args = TrainingArgs(dataset_json_file='./dataset.json') + train, test = build_dataset_from_file(training_args.dataset_json_file) + self.assertEqual(len(train) + len(test), len(dataset) + len(dataset2)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/trainers/hooks/test_lr_scheduler_hook.py b/tests/trainers/hooks/test_lr_scheduler_hook.py index cd28b055..432fb39a 100644 --- a/tests/trainers/hooks/test_lr_scheduler_hook.py +++ b/tests/trainers/hooks/test_lr_scheduler_hook.py @@ -105,6 +105,7 @@ class LrSchedulerHookTest(unittest.TestCase): train_dataloader = trainer._build_dataloader_with_dataset( trainer.train_dataset, **trainer.cfg.train.get('dataloader', {})) trainer.register_optimizers_hook() + trainer.register_processors() trainer._hooks = [ hook for hook in trainer._hooks if hook.__class__.__name__ not in ['CheckpointHook', 'TextLoggerHook', 'IterTimerHook'] @@ -177,6 +178,7 @@ class LrSchedulerHookTest(unittest.TestCase): train_dataloader = trainer._build_dataloader_with_dataset( trainer.train_dataset, **trainer.cfg.train.get('dataloader', {})) trainer.register_optimizers_hook() + trainer.register_processors() trainer._hooks = [ hook for hook in trainer._hooks if hook.__class__.__name__ not in ['CheckpointHook', 'TextLoggerHook', 'IterTimerHook'] @@ -365,6 +367,7 @@ class PlateauLrSchedulerHookTest(unittest.TestCase): trainer.train_dataloader = train_dataloader trainer.data_loader = train_dataloader trainer.register_optimizers_hook() + trainer.register_processors() trainer._hooks = [ hook for hook in trainer._hooks if hook.__class__.__name__ not in ['CheckpointHook', 'TextLoggerHook', 'IterTimerHook'] diff --git a/tests/trainers/hooks/test_optimizer_hook.py b/tests/trainers/hooks/test_optimizer_hook.py index b9899c36..ed0e202a 100644 --- a/tests/trainers/hooks/test_optimizer_hook.py +++ b/tests/trainers/hooks/test_optimizer_hook.py @@ -150,6 +150,7 @@ class TorchAMPOptimizerHookTest(unittest.TestCase): train_dataloader = trainer._build_dataloader_with_dataset( trainer.train_dataset, **trainer.cfg.train.get('dataloader', {})) trainer.register_optimizers_hook() + trainer.register_processors() trainer._hooks = [ hook for hook in trainer._hooks if hook.__class__.__name__ not in ['CheckpointHook', 'TextLoggerHook', 'IterTimerHook'] diff --git a/tests/trainers/test_trainer_with_nlp.py b/tests/trainers/test_trainer_with_nlp.py index ceb04e15..a736d4fa 100644 --- a/tests/trainers/test_trainer_with_nlp.py +++ b/tests/trainers/test_trainer_with_nlp.py @@ -9,6 +9,7 @@ import unittest import numpy as np import torch from packaging import version +from torch.utils.data import RandomSampler from modelscope.hub.snapshot_download import snapshot_download from modelscope.metainfo import Metrics @@ -204,12 +205,20 @@ class TestTrainerWithNlp(unittest.TestCase): cfg.preprocessor.val['label2id'] = {'0': 0, '1': 1} cfg.train.dataloader.batch_size_per_gpu = 2 cfg.train.hooks = [{ - 'type': 'BestCkptSaverHook', - 'interval': 1, - 'by_epoch': False, - 'metric_key': 'accuracy', - 'max_checkpoint_num': 4, - 'restore_best': True, + 'type': + 'BestCkptSaverHook', + 'interval': + 1, + 'by_epoch': + False, + 'output_dir': + os.path.join(self.tmp_dir, 'output_test_best'), + 'metric_key': + 'accuracy', + 'max_checkpoint_num': + 4, + 'restore_best': + True, }, { 'type': 'TextLoggerHook', 'interval': 1 @@ -270,7 +279,7 @@ class TestTrainerWithNlp(unittest.TestCase): os.path.join(self.tmp_dir, 'output', 'pytorch_model.bin'))) self.assertTrue( os.path.isfile( - os.path.join(self.tmp_dir, 'output_best', + os.path.join(self.tmp_dir, 'output_test_best', 'pytorch_model.bin'))) md51 = hashlib.md5( pathlib.Path( @@ -282,7 +291,7 @@ class TestTrainerWithNlp(unittest.TestCase): self.assertEqual(md51, md52) md51 = hashlib.md5( pathlib.Path( - os.path.join(self.tmp_dir, 'output_best', + os.path.join(self.tmp_dir, 'output_test_best', 'pytorch_model.bin')).read_bytes()).hexdigest() md52 = hashlib.md5( pathlib.Path( @@ -472,6 +481,34 @@ class TestTrainerWithNlp(unittest.TestCase): cache_path + '/pytorch_model.bin', saving_fn=saving_fn)) self.assertTrue(os.path.isfile(f'{tmp_dir}/predicts.txt')) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_trainer_with_custom_sampler(self): + tmp_dir = tempfile.TemporaryDirectory().name + if not os.path.exists(tmp_dir): + os.makedirs(tmp_dir) + + model_id = 'damo/nlp_structbert_sentence-similarity_chinese-tiny' + cache_path = snapshot_download(model_id) + model = SbertForSequenceClassification.from_pretrained(cache_path) + + class CustomSampler(RandomSampler): + + pass + + kwargs = dict( + cfg_file=os.path.join(cache_path, ModelFile.CONFIGURATION), + model=model, + train_dataset=self.dataset, + eval_dataset=self.dataset, + samplers=CustomSampler(self.dataset), + work_dir=self.tmp_dir) + + trainer = build_trainer(default_args=kwargs) + trainer.train() + self.assertTrue( + type(trainer.train_dataloader.sampler) == CustomSampler) + self.assertTrue(type(trainer.eval_dataloader.sampler) == CustomSampler) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_trainer_with_prediction(self): tmp_dir = tempfile.TemporaryDirectory().name diff --git a/tests/trainers/test_training_args.py b/tests/trainers/test_training_args.py index 6e4d306e..e8f6d8a2 100644 --- a/tests/trainers/test_training_args.py +++ b/tests/trainers/test_training_args.py @@ -1,8 +1,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import unittest -from modelscope.trainers.default_config import DEFAULT_CONFIG -from modelscope.trainers.training_args import CliArgumentParser, TrainingArgs +from modelscope import TrainingArgs +from modelscope.trainers.cli_argument_parser import CliArgumentParser from modelscope.utils.test_utils import test_level @@ -29,14 +29,14 @@ class TrainingArgsTest(unittest.TestCase): @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_flatten_args(self): - cfg = DEFAULT_CONFIG + training_args = TrainingArgs() input_args = [ '--optimizer_params', 'weight_decay=0.8,eps=1e-6,correct_bias=False', '--lr_scheduler_params', 'initial_lr=3e-5,niter_decay=1' ] - training_args = TrainingArgs.from_cli(input_args) - cfg = training_args(cfg) + training_args = training_args.parse_cli(input_args) + cfg, _ = training_args.to_config() self.assertAlmostEqual(cfg.train.optimizer.weight_decay, 0.8) self.assertAlmostEqual(cfg.train.optimizer.eps, 1e-6) self.assertFalse(cfg.train.optimizer.correct_bias) From bf2cd23aa2fa3ba444f4f6a086d0dcacd0b213f4 Mon Sep 17 00:00:00 2001 From: "xingjun.wang" Date: Sun, 14 May 2023 02:06:11 +0800 Subject: [PATCH 35/76] cancel a ut --- tests/pipelines/test_fill_mask.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pipelines/test_fill_mask.py b/tests/pipelines/test_fill_mask.py index 0e427464..2e380283 100644 --- a/tests/pipelines/test_fill_mask.py +++ b/tests/pipelines/test_fill_mask.py @@ -134,8 +134,9 @@ class FillMaskTest(unittest.TestCase, DemoCompatibilityCheck): f'\nori_text: {ori_text}\ninput: {test_input}\npipeline: ' f'{pipeline_ins(test_input)}\n') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_run_with_model_name(self): + # TODO: to be fixed in the future # veco pipeline_ins = pipeline(task=Tasks.fill_mask, model=self.model_id_veco) for language in ['zh', 'en']: From 0aaaaa13addda2df9f6e3175d05b4338958d9626 Mon Sep 17 00:00:00 2001 From: "lingcai.wl" Date: Sun, 14 May 2023 23:41:40 +0800 Subject: [PATCH 36/76] [to #49275037] remove demo check and fix service decoder Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12432683 --- modelscope/utils/demo_utils.py | 275 ------------------ modelscope/utils/service_utils.py | 3 + .../test_plugin_model.py | 3 +- .../test_abnormal_object_detection.py | 7 +- tests/pipelines/test_action_detection.py | 7 +- tests/pipelines/test_action_recognition.py | 7 +- .../test_adaint_image_color_enhance.py | 8 +- tests/pipelines/test_addr_mgeo.py | 7 +- tests/pipelines/test_addr_similarity.py | 7 +- tests/pipelines/test_animal_recognition.py | 7 +- tests/pipelines/test_arc_face_recognition.py | 7 +- .../pipelines/test_automatic_post_editing.py | 7 +- .../test_automatic_speech_recognition.py | 8 +- tests/pipelines/test_bad_image_detecting.py | 7 +- tests/pipelines/test_body_2d_keypoints.py | 7 +- tests/pipelines/test_body_3d_keypoints.py | 7 +- .../test_body_3d_keypoints_hdformer.py | 7 +- tests/pipelines/test_canmt_translation.py | 7 +- tests/pipelines/test_card_detection.py | 7 +- .../test_cartoon_stable_diffusion.py | 3 +- .../test_chinese_stable_diffusion.py | 3 +- .../pipelines/test_cmdssl_video_embedding.py | 7 +- tests/pipelines/test_content_check.py | 7 +- .../test_controllable_image_generation.py | 8 +- .../test_conversational_text_to_sql.py | 7 +- tests/pipelines/test_crowd_counting.py | 7 +- tests/pipelines/test_csanmt_translation.py | 7 +- tests/pipelines/test_damo_face_detection.py | 7 +- .../test_ddcolor_image_colorization.py | 7 +- .../test_ddpm_semantic_segmentation.py | 8 +- .../test_deeplpf_image_color_enhance.py | 7 +- .../test_dialog_intent_prediction.py | 7 +- tests/pipelines/test_dialog_modeling.py | 7 +- tests/pipelines/test_dialog_state_tracking.py | 7 +- .../test_diffusers_stable_diffusion.py | 3 +- .../pipelines/test_disco_guided_diffusion.py | 3 +- .../test_document_grounded_dialog_generate.py | 4 +- ...test_document_grounded_dialog_retrieval.py | 4 +- tests/pipelines/test_document_segmentation.py | 7 +- tests/pipelines/test_document_vl_embedding.py | 7 +- tests/pipelines/test_domain_classification.py | 7 +- .../test_efficient_diffusion_tuning.py | 16 +- .../test_extractive_summarization.py | 7 +- tests/pipelines/test_face_detection.py | 7 +- tests/pipelines/test_face_image_generation.py | 7 +- tests/pipelines/test_face_recognition.py | 7 +- .../test_face_recognition_onnx_fm.py | 7 +- .../test_face_recognition_onnx_ir.py | 7 +- tests/pipelines/test_face_recognition_ood.py | 7 +- tests/pipelines/test_face_reconstruction.py | 7 +- .../pipelines/test_faq_question_answering.py | 7 +- tests/pipelines/test_feature_extraction.py | 4 +- tests/pipelines/test_fid_dialogue.py | 7 +- tests/pipelines/test_fill_mask.py | 10 +- .../test_general_image_classification.py | 8 +- tests/pipelines/test_general_recognition.py | 7 +- .../test_generative_multi_modal_embedding.py | 7 +- .../pipelines/test_gridvlp_classification.py | 7 +- tests/pipelines/test_hand_detection.py | 3 +- .../pipelines/test_hicossl_video_embedding.py | 7 +- tests/pipelines/test_hitea_tasks.py | 7 +- tests/pipelines/test_image_body_reshaping.py | 7 +- tests/pipelines/test_image_color_enhance.py | 7 +- tests/pipelines/test_image_colorization.py | 7 +- tests/pipelines/test_image_debanding.py | 7 +- tests/pipelines/test_image_deblur.py | 7 +- tests/pipelines/test_image_defrcn_fewshot.py | 7 +- tests/pipelines/test_image_denoise.py | 7 +- .../pipelines/test_image_depth_estimation.py | 3 +- .../test_image_depth_estimation_bts.py | 7 +- .../test_image_driving_perception.py | 7 +- tests/pipelines/test_image_face_fusion.py | 7 +- tests/pipelines/test_image_human_parsing.py | 7 +- tests/pipelines/test_image_inpainting_sdv2.py | 7 +- .../test_image_instance_segmentation.py | 7 +- .../pipelines/test_image_layout_estimation.py | 3 +- tests/pipelines/test_image_matching.py | 3 +- tests/pipelines/test_image_matting.py | 7 +- .../test_image_mvs_depth_estimation.py | 3 +- .../test_image_open_vocabulary_detection.py | 8 +- .../test_image_panoptic_segmentation.py | 7 +- .../test_image_portrait_enhancement.py | 7 +- ...st_image_quality_assessment_degradation.py | 8 +- .../test_image_quality_assessment_man.py | 7 +- .../test_image_quality_assessment_mos.py | 7 +- tests/pipelines/test_image_reid_person.py | 7 +- tests/pipelines/test_image_restoration.py | 7 +- .../test_image_semantic_segmentation.py | 7 +- .../test_image_structured_model_probing.py | 4 +- tests/pipelines/test_image_style_transfer.py | 7 +- .../pipelines/test_image_super_resolution.py | 7 +- .../test_interactive_translation_pipeline.py | 7 +- .../pipelines/test_inverse_text_processing.py | 7 +- tests/pipelines/test_key_word_spotting.py | 7 +- ...est_language_guided_video_summarization.py | 8 +- .../pipelines/test_language_identification.py | 8 +- .../pipelines/test_license_plate_detection.py | 7 +- .../test_lineless_table_recognition.py | 7 +- tests/pipelines/test_live_category.py | 7 +- tests/pipelines/test_mask_face_recognition.py | 7 +- .../test_maskdino_instance_segmentation.py | 8 +- .../pipelines/test_mglm_text_summarization.py | 3 +- .../test_mobile_image_super_resolution.py | 8 +- tests/pipelines/test_motion_generation.py | 7 +- .../test_movie_scene_segmentation.py | 7 +- tests/pipelines/test_mplug_tasks.py | 7 +- ...st_msrresnetlite_video_super_resolution.py | 7 +- tests/pipelines/test_multi_modal_embedding.py | 7 +- ...t_multilingual_named_entity_recognition.py | 8 +- .../test_multilingual_word_segmentation.py | 7 +- .../test_named_entity_recognition.py | 7 +- tests/pipelines/test_nerf_recon_acc.py | 8 +- tests/pipelines/test_nli.py | 7 +- tests/pipelines/test_object_detecion_3d.py | 7 +- tests/pipelines/test_object_detection.py | 10 +- tests/pipelines/test_ocr_detection.py | 7 +- tests/pipelines/test_ocr_recognition.py | 7 +- tests/pipelines/test_ofa_tasks.py | 7 +- .../test_panorama_depth_estimation.py | 3 +- .../test_pedestrian_attribute_recognition.py | 8 +- tests/pipelines/test_person_image_cartoon.py | 7 +- .../test_pointcloud_sceneflow_estimation.py | 4 +- .../test_product_retrieval_embedding.py | 7 +- .../test_realtime_object_detection.py | 7 +- .../test_realtime_video_object_detection.py | 8 +- ...est_referring_video_object_segmentation.py | 8 +- tests/pipelines/test_relation_extraction.py | 7 +- tests/pipelines/test_salient_detection.py | 7 +- tests/pipelines/test_sentence_similarity.py | 7 +- .../test_sentiment_classification.py | 8 +- tests/pipelines/test_siamese_uie.py | 7 +- tests/pipelines/test_skin_retouching.py | 7 +- .../test_soonet_video_temporal_grounding.py | 4 +- tests/pipelines/test_speaker_verification.py | 3 +- tests/pipelines/test_speech_separation.py | 7 +- tests/pipelines/test_speech_signal_process.py | 7 +- tests/pipelines/test_table_recognition.py | 7 +- tests/pipelines/test_tbs_detection.py | 3 +- tests/pipelines/test_text2text_generation.py | 7 +- tests/pipelines/test_text_classification.py | 7 +- .../test_text_driven_segmentation.py | 4 - tests/pipelines/test_text_error_correction.py | 7 +- tests/pipelines/test_text_generation.py | 7 +- .../pipelines/test_text_to_image_synthesis.py | 7 +- tests/pipelines/test_text_to_speech.py | 8 +- .../pipelines/test_text_to_video_synthesis.py | 7 +- .../pipelines/test_tinymog_face_detection.py | 7 +- .../pipelines/test_tinynas_classification.py | 7 +- tests/pipelines/test_tinynas_detection.py | 7 +- .../pipelines/test_traffic_sign_detection.py | 7 +- .../pipelines/test_translation_evaluation.py | 3 +- .../test_translation_quality_estimation.py | 8 +- tests/pipelines/test_unifold.py | 3 +- tests/pipelines/test_universal_matting.py | 7 +- .../test_user_satisfaction_estimation.py | 8 +- tests/pipelines/test_video_category.py | 7 +- tests/pipelines/test_video_colorization.py | 7 +- tests/pipelines/test_video_deinterlace.py | 7 +- .../pipelines/test_video_depth_estimation.py | 3 +- .../test_video_frame_interpolation.py | 7 +- .../test_video_instance_segmentation.py | 7 +- .../test_video_multi_modal_embedding.py | 7 +- .../test_video_multi_object_tracking.py | 7 +- .../test_video_object_segmentation.py | 3 +- .../test_video_panoptic_segmentation.py | 7 +- .../test_video_single_object_tracking.py | 7 +- tests/pipelines/test_video_stabilization.py | 7 +- tests/pipelines/test_video_summarization.py | 7 +- .../pipelines/test_video_super_resolution.py | 7 +- tests/pipelines/test_vidt_face.py | 3 +- tests/pipelines/test_vidt_logo.py | 3 +- tests/pipelines/test_virtual_try_on.py | 7 +- .../pipelines/test_vision_efficient_tuning.py | 38 +-- tests/pipelines/test_vision_middleware.py | 3 +- tests/pipelines/test_vop_retrieval.py | 3 +- tests/pipelines/test_vop_retrieval_sebias.py | 3 +- .../pipelines/test_vop_retrieval_separtial.py | 3 +- tests/pipelines/test_vop_retrieval_seproj.py | 3 +- ...test_wenet_automatic_speech_recognition.py | 4 +- tests/pipelines/test_word_segmentation.py | 7 +- .../test_zero_shot_classification.py | 7 +- 181 files changed, 187 insertions(+), 1286 deletions(-) delete mode 100644 modelscope/utils/demo_utils.py diff --git a/modelscope/utils/demo_utils.py b/modelscope/utils/demo_utils.py deleted file mode 100644 index 99e61d45..00000000 --- a/modelscope/utils/demo_utils.py +++ /dev/null @@ -1,275 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. - -import io - -import json - -from modelscope.outputs import OutputKeys -from modelscope.pipelines import pipeline -from modelscope.utils.constant import Tasks, TasksIODescriptions -from modelscope.utils.service_utils import NumpyEncoder - -TASKS_INPUT_TEMPLATES = { - # vision tasks - Tasks.image_portrait_stylization: TasksIODescriptions.image_to_image, - Tasks.portrait_matting: TasksIODescriptions.image_to_image, - Tasks.skin_retouching: TasksIODescriptions.image_to_image, - Tasks.image_captioning: TasksIODescriptions.image_to_text, - Tasks.image_denoising: TasksIODescriptions.image_to_image, - Tasks.image_portrait_enhancement: TasksIODescriptions.image_to_image, - Tasks.image_super_resolution: TasksIODescriptions.image_to_image, - Tasks.image_colorization: TasksIODescriptions.image_to_image, - Tasks.image_color_enhancement: TasksIODescriptions.image_to_image, - Tasks.face_image_generation: TasksIODescriptions.seed_to_image, - Tasks.image_style_transfer: TasksIODescriptions.images_to_image, - Tasks.image_segmentation: TasksIODescriptions.image_to_text, - Tasks.image_object_detection: TasksIODescriptions.image_to_text, - - # not tested - Tasks.image_classification: TasksIODescriptions.image_to_text, - Tasks.ocr_detection: TasksIODescriptions.image_to_text, - Tasks.ocr_recognition: TasksIODescriptions.image_to_text, - Tasks.body_2d_keypoints: TasksIODescriptions.image_to_text, - Tasks.vision_efficient_tuning: TasksIODescriptions.image_to_text, - - # nlp tasks - Tasks.text_classification: TasksIODescriptions.text_to_text, - Tasks.text_generation: TasksIODescriptions.text_to_text, - Tasks.word_segmentation: TasksIODescriptions.text_to_text, - Tasks.text_error_correction: TasksIODescriptions.text_to_text, - Tasks.named_entity_recognition: TasksIODescriptions.text_to_text, - Tasks.sentiment_classification: TasksIODescriptions.text_to_text, - - # audio tasks - Tasks.text_to_speech: TasksIODescriptions.text_to_speech, - Tasks.auto_speech_recognition: TasksIODescriptions.speech_to_text, - Tasks.keyword_spotting: TasksIODescriptions.speech_to_text, - Tasks.acoustic_noise_suppression: TasksIODescriptions.speech_to_speech, - Tasks.acoustic_echo_cancellation: TasksIODescriptions.speeches_to_speech, - - # multi-modal - Tasks.visual_grounding: TasksIODescriptions.visual_grounding, - Tasks.visual_question_answering: - TasksIODescriptions.visual_question_answering, - Tasks.visual_entailment: TasksIODescriptions.visual_entailment, - Tasks.generative_multi_modal_embedding: - TasksIODescriptions.generative_multi_modal_embedding, - - # new tasks - Tasks.virtual_try_on: TasksIODescriptions.images_to_image, - - # TODO(lingcai.wl): support more tasks and implement corresponding example -} - -INPUT_EXAMPLES = { - # Must align with task schema defined in the Widget section of model card= - # cv - TasksIODescriptions.image_to_image: { - 'inputs': [ - 'https://modelscope.oss-cn-beijing.aliyuncs.com/test/images/image_cartoon.png' - ], - 'urlPaths': { - 'outUrls': [{ - 'outputKey': OutputKeys.OUTPUT_IMG, - 'fileType': 'png' - }] - } - }, - TasksIODescriptions.images_to_image: { - 'inputs': [ - 'https://modelscope.oss-cn-beijing.aliyuncs.com/demo/image-style-transfer/style_transfer_content.jpg', - 'https://modelscope.oss-cn-beijing.aliyuncs.com/demo/image-style-transfer/style_transfer_style.jpg' - ], - 'urlPaths': { - 'outUrls': [{ - 'outputKey': OutputKeys.OUTPUT_IMG, - 'fileType': 'png' - }] - } - }, - TasksIODescriptions.image_to_text: { - 'inputs': [ - 'https://modelscope.oss-cn-beijing.aliyuncs.com/test/images/image_cartoon.png' - ], - 'urlPaths': {} - }, - # nlp - TasksIODescriptions.text_to_text: { - 'inputs': ['test'], - 'urlPaths': {} - }, - - # audio - TasksIODescriptions.speech_to_text: { - 'inputs': [ - 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example.wav' - ], - 'urlPaths': {} - }, - TasksIODescriptions.text_to_speech: { - 'inputs': ['北京今天天气怎么样'], - 'urlPaths': { - 'outUrls': [{ - 'outputKey': OutputKeys.OUTPUT_PCM, - 'fileType': 'pcm' - }] - } - }, - TasksIODescriptions.speeches_to_speech: { - 'inputs': [ - 'http://225252-file.oss-cn-hangzhou-zmf.aliyuncs.com/maas_demo/nearend_mic.wav', - 'http://225252-file.oss-cn-hangzhou-zmf.aliyuncs.com/maas_demo/nearend_speech.wav' - ], - 'urlPaths': { - 'outUrls': [{ - 'outputKey': OutputKeys.OUTPUT_PCM, - 'fileType': 'pcm' - }] - } - }, - TasksIODescriptions.speech_to_speech: { - 'inputs': [ - 'http://225252-file.oss-cn-hangzhou-zmf.aliyuncs.com/maas_demo/speech_with_noise.wav' - ], - 'urlPaths': { - 'outUrls': [{ - 'outputKey': OutputKeys.OUTPUT_PCM, - 'fileType': 'pcm' - }] - } - }, - - # multi modal - TasksIODescriptions.visual_grounding: { - 'task': - Tasks.visual_grounding, - 'inputs': [ - 'http://xingchen-data.oss-cn-zhangjiakou.aliyuncs.com/maas/visual-grounding/visual_grounding.png', - 'a blue turtle-like pokemon with round head' - ], - 'urlPaths': { - 'inUrls': [{ - 'name': 'image' - }, { - 'name': 'text' - }] - } - }, - TasksIODescriptions.visual_question_answering: { - 'task': - Tasks.visual_question_answering, - 'inputs': [ - 'http://225252-file.oss-cn-hangzhou-zmf.aliyuncs.com/maas_demo/visual_question_answering.png', - 'what is grown on the plant?' - ], - 'urlPaths': { - 'inUrls': [{ - 'name': 'image' - }, { - 'name': 'text' - }], - 'outUrls': [{ - 'outputKey': 'text' - }] - } - }, - TasksIODescriptions.visual_entailment: { - 'task': - Tasks.visual_entailment, - 'inputs': [ - 'http://xingchen-data.oss-cn-zhangjiakou.aliyuncs.com/maas/visual-entailment/visual_entailment.jpg', - 'there are two birds.', 'test' - ], - 'urlPaths': { - 'inUrls': [{ - 'name': 'image' - }, { - 'name': 'text' - }], - 'outUrls': [{}] - } - }, - TasksIODescriptions.generative_multi_modal_embedding: { - 'task': - Tasks.generative_multi_modal_embedding, - 'inputs': [ - 'http://clip-multimodal.oss-cn-beijing.aliyuncs.com/lingchen/demo/dogs.jpg', - 'dogs playing in the grass' - ], - 'urlPaths': { - 'inUrls': [{ - 'name': 'image' - }, { - 'name': 'text' - }], - 'outUrls': [{}] - } - }, -} - - -class DemoCompatibilityCheck(object): - - def compatibility_check(self): - if self.task not in TASKS_INPUT_TEMPLATES: - print('task is not supported in demo service so far') - return False - if TASKS_INPUT_TEMPLATES[self.task] not in INPUT_EXAMPLES: - print('no example input for this task') - return False - - print('testing demo: ', self.task, self.model_id) - test_pipline = pipeline(self.task, self.model_id) - req = INPUT_EXAMPLES[TASKS_INPUT_TEMPLATES[self.task]] - inputs = preprocess(req) - params = req.get('parameters', {}) - # modelscope inference - if params != {}: - output = test_pipline(inputs, **params) - else: - output = test_pipline(inputs) - json.dumps(output, cls=NumpyEncoder) - result = postprocess(req, output) - print(result) - return True - - -def preprocess(req): - in_urls = req.get('urlPaths').get('inUrls') - if len(req['inputs']) == 1: - inputs = req['inputs'][0] - else: - inputs = tuple(req['inputs']) - if in_urls is None or len(in_urls) == 0: - return inputs - - inputs_dict = {} - for i, in_url in enumerate(in_urls): - input_name = in_url.get('name') - if input_name is None or input_name == '': - return inputs - inputs_dict[input_name] = req['inputs'][i] - return inputs_dict - - -def postprocess(req, resp): - out_urls = req.get('urlPaths').get('outUrls') - if out_urls is None or len(out_urls) == 0: - return resp - new_resp = resp - if isinstance(resp, str): - new_resp = json.loads(resp) - for out_url in out_urls: - output_key = out_url['outputKey'] - file_type = out_url['fileType'] - new_resp.get(output_key) - if file_type == 'png' or file_type == 'jpg': - content = new_resp.get(output_key) - import cv2 - _, img_encode = cv2.imencode('.' + file_type, content) - img_bytes = img_encode.tobytes() - return type(img_bytes) - else: - out_mem_file = io.BytesIO() - out_mem_file.write(new_resp.get(output_key)) - return type(out_mem_file) diff --git a/modelscope/utils/service_utils.py b/modelscope/utils/service_utils.py index e20988f7..8f7ca42d 100644 --- a/modelscope/utils/service_utils.py +++ b/modelscope/utils/service_utils.py @@ -156,6 +156,7 @@ def service_data_decoder(task, data): return input_data elif isinstance(input_type, dict): input_data = {} + data = json.loads(data) for key, val in input_type.items(): if val == InputType.IMAGE: input_data[key] = decode_base64_to_image(data[key]) @@ -163,6 +164,8 @@ def service_data_decoder(task, data): input_data[key] = decode_base64_to_binary(data[key])[0] elif val == InputType.TEXT: input_data[key] = data[key] + else: + return data return input_data diff --git a/tests/pipelines/plugin_remote_pipelines/test_plugin_model.py b/tests/pipelines/plugin_remote_pipelines/test_plugin_model.py index 43d840ea..71b9e64f 100644 --- a/tests/pipelines/plugin_remote_pipelines/test_plugin_model.py +++ b/tests/pipelines/plugin_remote_pipelines/test_plugin_model.py @@ -3,12 +3,11 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.plugins import PluginsManager from modelscope.utils.test_utils import test_level -class PluginModelTest(unittest.TestCase, DemoCompatibilityCheck): +class PluginModelTest(unittest.TestCase): def setUp(self): self.package = 'adaseq' diff --git a/tests/pipelines/test_abnormal_object_detection.py b/tests/pipelines/test_abnormal_object_detection.py index fbce51c6..c6264069 100644 --- a/tests/pipelines/test_abnormal_object_detection.py +++ b/tests/pipelines/test_abnormal_object_detection.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): +class ObjectDetectionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_object_detection @@ -20,10 +19,6 @@ class ObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): result = object_detect(input_location) print(result) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_action_detection.py b/tests/pipelines/test_action_detection.py index ae7e60b1..d724c81a 100644 --- a/tests/pipelines/test_action_detection.py +++ b/tests/pipelines/test_action_detection.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ActionDetectionTest(unittest.TestCase, DemoCompatibilityCheck): +class ActionDetectionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.action_detection @@ -20,10 +19,6 @@ class ActionDetectionTest(unittest.TestCase, DemoCompatibilityCheck): 'data/test/videos/action_detection_test_video.mp4') print('action detection results:', result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_action_recognition.py b/tests/pipelines/test_action_recognition.py index 292eb238..9d0c6175 100644 --- a/tests/pipelines/test_action_recognition.py +++ b/tests/pipelines/test_action_recognition.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ActionRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): +class ActionRecognitionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.action_recognition @@ -37,10 +36,6 @@ class ActionRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): 'data/test/videos/action_recognition_test_video.mp4') print('pst recognition results:', result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_adaint_image_color_enhance.py b/tests/pipelines/test_adaint_image_color_enhance.py index e36a85ec..f0efef5f 100644 --- a/tests/pipelines/test_adaint_image_color_enhance.py +++ b/tests/pipelines/test_adaint_image_color_enhance.py @@ -9,11 +9,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class AdaIntImageColorEnhanceTest(unittest.TestCase, DemoCompatibilityCheck): +class AdaIntImageColorEnhanceTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_adaint_image-color-enhance-models' @@ -40,11 +39,6 @@ class AdaIntImageColorEnhanceTest(unittest.TestCase, DemoCompatibilityCheck): self.pipeline_inference(img_color_enhance, 'data/test/images/image_color_enhance.png') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - @unittest.skipIf(not torch.cuda.is_available(), 'cuda unittest only') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_addr_mgeo.py b/tests/pipelines/test_addr_mgeo.py index d630b857..e678d285 100644 --- a/tests/pipelines/test_addr_mgeo.py +++ b/tests/pipelines/test_addr_mgeo.py @@ -8,12 +8,11 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import TextClassificationPipeline from modelscope.preprocessors import TextClassificationTransformersPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.regress_test_utils import IgnoreKeyFn, MsRegressTool from modelscope.utils.test_utils import test_level -class MGeoTest(unittest.TestCase, DemoCompatibilityCheck): +class MGeoTest(unittest.TestCase): multi_modal_inputs = { 'source_sentence': ['杭州余杭东方未来学校附近世纪华联商场(金家渡北苑店)'], @@ -117,10 +116,6 @@ class MGeoTest(unittest.TestCase, DemoCompatibilityCheck): pipeline_ins = pipeline(task=task, model=model) print(pipeline_ins(input=inputs)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_addr_similarity.py b/tests/pipelines/test_addr_similarity.py index 8c1f93c9..ecc879eb 100644 --- a/tests/pipelines/test_addr_similarity.py +++ b/tests/pipelines/test_addr_similarity.py @@ -8,12 +8,11 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import TextClassificationPipeline from modelscope.preprocessors import TextClassificationTransformersPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.regress_test_utils import IgnoreKeyFn, MsRegressTool from modelscope.utils.test_utils import test_level -class AddrSimilarityTest(unittest.TestCase, DemoCompatibilityCheck): +class AddrSimilarityTest(unittest.TestCase): sentence1 = '阿里巴巴西溪园区' sentence2 = '文一西路阿里巴巴' @@ -37,10 +36,6 @@ class AddrSimilarityTest(unittest.TestCase, DemoCompatibilityCheck): task=Tasks.text_classification, model=self.model_id) print(pipeline_ins(input=(self.sentence1, self.sentence2))) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_animal_recognition.py b/tests/pipelines/test_animal_recognition.py index eb9f92e6..57937770 100644 --- a/tests/pipelines/test_animal_recognition.py +++ b/tests/pipelines/test_animal_recognition.py @@ -4,11 +4,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class AnimalRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): +class AnimalRecognitionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.animal_recognition @@ -21,10 +20,6 @@ class AnimalRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): result = animal_recognition('data/test/images/dogs.jpg') print(result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_arc_face_recognition.py b/tests/pipelines/test_arc_face_recognition.py index fa17dd91..481b9f33 100644 --- a/tests/pipelines/test_arc_face_recognition.py +++ b/tests/pipelines/test_arc_face_recognition.py @@ -6,11 +6,10 @@ import numpy as np from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class FaceRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): +class FaceRecognitionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.face_recognition @@ -31,10 +30,6 @@ class FaceRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): sim = np.dot(emb1[0], emb2[0]) print(f'Cos similarity={sim:.3f}, img1:{img1} img2:{img2}') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_automatic_post_editing.py b/tests/pipelines/test_automatic_post_editing.py index da09851c..190ff788 100644 --- a/tests/pipelines/test_automatic_post_editing.py +++ b/tests/pipelines/test_automatic_post_editing.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class AutomaticPostEditingTest(unittest.TestCase, DemoCompatibilityCheck): +class AutomaticPostEditingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.translation @@ -21,10 +20,6 @@ class AutomaticPostEditingTest(unittest.TestCase, DemoCompatibilityCheck): pipeline_ins = pipeline(self.task, model=self.model_id) print(pipeline_ins(input=inputs)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_automatic_speech_recognition.py b/tests/pipelines/test_automatic_speech_recognition.py index dc624f29..6014438e 100644 --- a/tests/pipelines/test_automatic_speech_recognition.py +++ b/tests/pipelines/test_automatic_speech_recognition.py @@ -10,7 +10,6 @@ import soundfile from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import ColorCodes, Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import download_and_untar, test_level @@ -26,8 +25,7 @@ TFRECORD_TESTSETS_FILE = 'tfrecord.tar.gz' TFRECORD_TESTSETS_URL = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/datasets/tfrecord.tar.gz' -class AutomaticSpeechRecognitionTest(unittest.TestCase, - DemoCompatibilityCheck): +class AutomaticSpeechRecognitionTest(unittest.TestCase): action_info = { 'test_run_with_wav_pytorch': { 'checking_item': OutputKeys.TEXT, @@ -457,10 +455,6 @@ class AutomaticSpeechRecognitionTest(unittest.TestCase, logger.info(ColorCodes.MAGENTA + str(rec_result) + ColorCodes.END) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_bad_image_detecting.py b/tests/pipelines/test_bad_image_detecting.py index 728da8d1..05954f7b 100644 --- a/tests/pipelines/test_bad_image_detecting.py +++ b/tests/pipelines/test_bad_image_detecting.py @@ -7,11 +7,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.cv import BadImageDetecingPipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class BadImageDetectingTest(unittest.TestCase, DemoCompatibilityCheck): +class BadImageDetectingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.bad_image_detecting @@ -58,10 +57,6 @@ class BadImageDetectingTest(unittest.TestCase, DemoCompatibilityCheck): print('pipeline: the out_label is {}'.format(labels)) print('pipeline: the out_score is {}'.format(scores)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_body_2d_keypoints.py b/tests/pipelines/test_body_2d_keypoints.py index 5d90cbf0..25d8fa55 100644 --- a/tests/pipelines/test_body_2d_keypoints.py +++ b/tests/pipelines/test_body_2d_keypoints.py @@ -8,11 +8,10 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import draw_keypoints -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class Body2DKeypointsTest(unittest.TestCase, DemoCompatibilityCheck): +class Body2DKeypointsTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.body_2d_keypoints @@ -34,10 +33,6 @@ class Body2DKeypointsTest(unittest.TestCase, DemoCompatibilityCheck): body_2d_keypoints = pipeline(self.task, model=self.model_id) self.pipeline_inference(body_2d_keypoints, Image.open(self.test_image)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_body_3d_keypoints.py b/tests/pipelines/test_body_3d_keypoints.py index 6f73a243..33228022 100644 --- a/tests/pipelines/test_body_3d_keypoints.py +++ b/tests/pipelines/test_body_3d_keypoints.py @@ -8,11 +8,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class Body3DKeypointsTest(unittest.TestCase, DemoCompatibilityCheck): +class Body3DKeypointsTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_canonical_body-3d-keypoints_video' @@ -41,10 +40,6 @@ class Body3DKeypointsTest(unittest.TestCase, DemoCompatibilityCheck): % (self.test_video)) self.pipeline_inference(body_3d_keypoints, pipeline_input=cap) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_body_3d_keypoints_hdformer.py b/tests/pipelines/test_body_3d_keypoints_hdformer.py index 2ebbc95b..e86f247f 100644 --- a/tests/pipelines/test_body_3d_keypoints_hdformer.py +++ b/tests/pipelines/test_body_3d_keypoints_hdformer.py @@ -8,11 +8,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class Body3DKeypointsHDFormerTest(unittest.TestCase, DemoCompatibilityCheck): +class Body3DKeypointsHDFormerTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_hdformer_body-3d-keypoints_video' @@ -41,10 +40,6 @@ class Body3DKeypointsHDFormerTest(unittest.TestCase, DemoCompatibilityCheck): % (self.test_video)) self.pipeline_inference(body_3d_keypoints, pipeline_input=cap) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_canmt_translation.py b/tests/pipelines/test_canmt_translation.py index e3bce5d9..31e57040 100644 --- a/tests/pipelines/test_canmt_translation.py +++ b/tests/pipelines/test_canmt_translation.py @@ -8,11 +8,10 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import CanmtTranslationPipeline from modelscope.preprocessors import CanmtTranslationPreprocessor, Preprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class CanmtTranslationTest(unittest.TestCase, DemoCompatibilityCheck): +class CanmtTranslationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.competency_aware_translation @@ -59,10 +58,6 @@ class CanmtTranslationTest(unittest.TestCase, DemoCompatibilityCheck): pipeline_ins = pipeline(task=self.task) print(pipeline_ins(self.input)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_card_detection.py b/tests/pipelines/test_card_detection.py index d913f494..676fb786 100644 --- a/tests/pipelines/test_card_detection.py +++ b/tests/pipelines/test_card_detection.py @@ -8,11 +8,10 @@ from modelscope.msdatasets import MsDataset from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import draw_card_detection_result -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class CardDetectionTest(unittest.TestCase, DemoCompatibilityCheck): +class CardDetectionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.card_detection @@ -57,10 +56,6 @@ class CardDetectionTest(unittest.TestCase, DemoCompatibilityCheck): result = card_detection(img_path) self.show_result(img_path, result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_cartoon_stable_diffusion.py b/tests/pipelines/test_cartoon_stable_diffusion.py index 751c7ea8..6a91ed44 100644 --- a/tests/pipelines/test_cartoon_stable_diffusion.py +++ b/tests/pipelines/test_cartoon_stable_diffusion.py @@ -6,11 +6,10 @@ import cv2 from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class CartoonStableDiffusionTest(unittest.TestCase, DemoCompatibilityCheck): +class CartoonStableDiffusionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.text_to_image_synthesis diff --git a/tests/pipelines/test_chinese_stable_diffusion.py b/tests/pipelines/test_chinese_stable_diffusion.py index bd6d74aa..05207ddb 100644 --- a/tests/pipelines/test_chinese_stable_diffusion.py +++ b/tests/pipelines/test_chinese_stable_diffusion.py @@ -6,11 +6,10 @@ import cv2 from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ChineseStableDiffusionTest(unittest.TestCase, DemoCompatibilityCheck): +class ChineseStableDiffusionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.text_to_image_synthesis diff --git a/tests/pipelines/test_cmdssl_video_embedding.py b/tests/pipelines/test_cmdssl_video_embedding.py index 5807c075..9e176cf2 100644 --- a/tests/pipelines/test_cmdssl_video_embedding.py +++ b/tests/pipelines/test_cmdssl_video_embedding.py @@ -4,11 +4,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class CMDSSLVideoEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck): +class CMDSSLVideoEmbeddingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_embedding @@ -22,10 +21,6 @@ class CMDSSLVideoEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck): print(f'video embedding output: {result}.') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_content_check.py b/tests/pipelines/test_content_check.py index c68af257..39a791a0 100644 --- a/tests/pipelines/test_content_check.py +++ b/tests/pipelines/test_content_check.py @@ -4,11 +4,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ContentCheckTest(unittest.TestCase, DemoCompatibilityCheck): +class ContentCheckTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_classification @@ -20,10 +19,6 @@ class ContentCheckTest(unittest.TestCase, DemoCompatibilityCheck): result = content_check_func('data/test/images/content_check.jpg') print(result) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_controllable_image_generation.py b/tests/pipelines/test_controllable_image_generation.py index a5cff66c..7d6b03ce 100644 --- a/tests/pipelines/test_controllable_image_generation.py +++ b/tests/pipelines/test_controllable_image_generation.py @@ -10,12 +10,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.cv import ControllableImageGenerationPipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ControllableImageGenerationTest(unittest.TestCase, - DemoCompatibilityCheck): +class ControllableImageGenerationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.controllable_image_generation @@ -68,10 +66,6 @@ class ControllableImageGenerationTest(unittest.TestCase, print( 'pipeline: the output image path is {}'.format(output_image_path)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_conversational_text_to_sql.py b/tests/pipelines/test_conversational_text_to_sql.py index 17fffcaf..a7e15dcc 100644 --- a/tests/pipelines/test_conversational_text_to_sql.py +++ b/tests/pipelines/test_conversational_text_to_sql.py @@ -8,13 +8,12 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import ConversationalTextToSqlPipeline from modelscope.preprocessors import ConversationalTextToSqlPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.nlp.space_T_en.utils import \ text2sql_tracking_and_print_results from modelscope.utils.test_utils import test_level -class ConversationalTextToSql(unittest.TestCase, DemoCompatibilityCheck): +class ConversationalTextToSql(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.table_question_answering @@ -67,10 +66,6 @@ class ConversationalTextToSql(unittest.TestCase, DemoCompatibilityCheck): pipelines = [pipeline(task=self.task, model=self.model_id)] text2sql_tracking_and_print_results(self.test_case, pipelines) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_crowd_counting.py b/tests/pipelines/test_crowd_counting.py index 4e15cfca..be14f29e 100644 --- a/tests/pipelines/test_crowd_counting.py +++ b/tests/pipelines/test_crowd_counting.py @@ -8,14 +8,13 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import numpy_to_cv2img -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import test_level logger = get_logger() -class CrowdCountingTest(unittest.TestCase, DemoCompatibilityCheck): +class CrowdCountingTest(unittest.TestCase): def setUp(self) -> None: self.input_location = 'data/test/images/crowd_counting.jpg' @@ -56,10 +55,6 @@ class CrowdCountingTest(unittest.TestCase, DemoCompatibilityCheck): else: raise ValueError('process error') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_csanmt_translation.py b/tests/pipelines/test_csanmt_translation.py index d989a6c4..03545fc5 100644 --- a/tests/pipelines/test_csanmt_translation.py +++ b/tests/pipelines/test_csanmt_translation.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class TranslationTest(unittest.TestCase, DemoCompatibilityCheck): +class TranslationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.translation @@ -90,10 +89,6 @@ class TranslationTest(unittest.TestCase, DemoCompatibilityCheck): pipeline_ins = pipeline(self.task) print(pipeline_ins(input=inputs)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_damo_face_detection.py b/tests/pipelines/test_damo_face_detection.py index 8bd1e009..44578c3e 100644 --- a/tests/pipelines/test_damo_face_detection.py +++ b/tests/pipelines/test_damo_face_detection.py @@ -7,11 +7,10 @@ import cv2 from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import draw_face_detection_result -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class FaceDetectionTest(unittest.TestCase, DemoCompatibilityCheck): +class FaceDetectionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.face_detection @@ -36,10 +35,6 @@ class FaceDetectionTest(unittest.TestCase, DemoCompatibilityCheck): result = face_detection(img_path) self.show_result(img_path, result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_ddcolor_image_colorization.py b/tests/pipelines/test_ddcolor_image_colorization.py index e1876329..5d752452 100644 --- a/tests/pipelines/test_ddcolor_image_colorization.py +++ b/tests/pipelines/test_ddcolor_image_colorization.py @@ -11,11 +11,10 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.pipelines.cv import DDColorImageColorizationPipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class DDColorImageColorizationTest(unittest.TestCase, DemoCompatibilityCheck): +class DDColorImageColorizationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_colorization @@ -52,10 +51,6 @@ class DDColorImageColorizationTest(unittest.TestCase, DemoCompatibilityCheck): image_colorization = pipeline(Tasks.image_colorization) self.pipeline_inference(image_colorization, self.test_image) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_ddpm_semantic_segmentation.py b/tests/pipelines/test_ddpm_semantic_segmentation.py index a5303098..ad35e069 100644 --- a/tests/pipelines/test_ddpm_semantic_segmentation.py +++ b/tests/pipelines/test_ddpm_semantic_segmentation.py @@ -5,12 +5,10 @@ import torch from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class DDPMImageSemanticSegmentationTest(unittest.TestCase, - DemoCompatibilityCheck): +class DDPMImageSemanticSegmentationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_segmentation @@ -28,10 +26,6 @@ class DDPMImageSemanticSegmentationTest(unittest.TestCase, else: raise ValueError('process error') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_deeplpf_image_color_enhance.py b/tests/pipelines/test_deeplpf_image_color_enhance.py index 08b1a357..87b709fe 100644 --- a/tests/pipelines/test_deeplpf_image_color_enhance.py +++ b/tests/pipelines/test_deeplpf_image_color_enhance.py @@ -8,11 +8,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class DeepLPFImageColorEnhanceTest(unittest.TestCase, DemoCompatibilityCheck): +class DeepLPFImageColorEnhanceTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_deeplpfnet_image-color-enhance-models' @@ -37,10 +36,6 @@ class DeepLPFImageColorEnhanceTest(unittest.TestCase, DemoCompatibilityCheck): self.pipeline_inference(img_color_enhance, 'data/test/images/image_color_enhance.png') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_dialog_intent_prediction.py b/tests/pipelines/test_dialog_intent_prediction.py index 2ee46388..5a1729b8 100644 --- a/tests/pipelines/test_dialog_intent_prediction.py +++ b/tests/pipelines/test_dialog_intent_prediction.py @@ -8,11 +8,10 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import DialogIntentPredictionPipeline from modelscope.preprocessors import DialogIntentPredictionPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class DialogIntentPredictionTest(unittest.TestCase, DemoCompatibilityCheck): +class DialogIntentPredictionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.task_oriented_conversation @@ -68,10 +67,6 @@ class DialogIntentPredictionTest(unittest.TestCase, DemoCompatibilityCheck): for my_pipeline, item in list(zip(pipelines, self.test_case)): print(my_pipeline(item)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_dialog_modeling.py b/tests/pipelines/test_dialog_modeling.py index 6b6259ce..202951a5 100644 --- a/tests/pipelines/test_dialog_modeling.py +++ b/tests/pipelines/test_dialog_modeling.py @@ -10,11 +10,10 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import DialogModelingPipeline from modelscope.preprocessors import DialogModelingPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class DialogModelingTest(unittest.TestCase, DemoCompatibilityCheck): +class DialogModelingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.task_oriented_conversation @@ -148,10 +147,6 @@ class DialogModelingTest(unittest.TestCase, DemoCompatibilityCheck): pipelines = [pipeline(task=self.task)] self.generate_and_print_dialog_response(pipelines) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_dialog_state_tracking.py b/tests/pipelines/test_dialog_state_tracking.py index 6cdd5ee7..e7f72b83 100644 --- a/tests/pipelines/test_dialog_state_tracking.py +++ b/tests/pipelines/test_dialog_state_tracking.py @@ -8,13 +8,12 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import DialogStateTrackingPipeline from modelscope.preprocessors import DialogStateTrackingPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.nlp.space.utils_dst import \ tracking_and_print_dialog_states from modelscope.utils.test_utils import test_level -class DialogStateTrackingTest(unittest.TestCase, DemoCompatibilityCheck): +class DialogStateTrackingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.task_oriented_conversation @@ -119,10 +118,6 @@ class DialogStateTrackingTest(unittest.TestCase, DemoCompatibilityCheck): pipelines = [pipeline(task=self.task, model=self.model_id)] tracking_and_print_dialog_states(self.test_case, pipelines) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_diffusers_stable_diffusion.py b/tests/pipelines/test_diffusers_stable_diffusion.py index 98c4862a..eef677fc 100644 --- a/tests/pipelines/test_diffusers_stable_diffusion.py +++ b/tests/pipelines/test_diffusers_stable_diffusion.py @@ -6,11 +6,10 @@ import cv2 from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class DiffusersStableDiffusionTest(unittest.TestCase, DemoCompatibilityCheck): +class DiffusersStableDiffusionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.text_to_image_synthesis diff --git a/tests/pipelines/test_disco_guided_diffusion.py b/tests/pipelines/test_disco_guided_diffusion.py index d7be7292..f3fd668b 100644 --- a/tests/pipelines/test_disco_guided_diffusion.py +++ b/tests/pipelines/test_disco_guided_diffusion.py @@ -5,11 +5,10 @@ import cv2 from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class DiscoGuidedDiffusionTest(unittest.TestCase, DemoCompatibilityCheck): +class DiscoGuidedDiffusionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.text_to_image_synthesis diff --git a/tests/pipelines/test_document_grounded_dialog_generate.py b/tests/pipelines/test_document_grounded_dialog_generate.py index da23fe19..b08a07fa 100644 --- a/tests/pipelines/test_document_grounded_dialog_generate.py +++ b/tests/pipelines/test_document_grounded_dialog_generate.py @@ -9,12 +9,10 @@ from modelscope.pipelines import pipeline from modelscope.preprocessors.nlp import \ DocumentGroundedDialogGeneratePreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class DocumentGroundedDialogGenerateTest(unittest.TestCase, - DemoCompatibilityCheck): +class DocumentGroundedDialogGenerateTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.document_grounded_dialog_generate diff --git a/tests/pipelines/test_document_grounded_dialog_retrieval.py b/tests/pipelines/test_document_grounded_dialog_retrieval.py index 6bcca369..48a63087 100644 --- a/tests/pipelines/test_document_grounded_dialog_retrieval.py +++ b/tests/pipelines/test_document_grounded_dialog_retrieval.py @@ -9,12 +9,10 @@ from modelscope.pipelines import pipeline from modelscope.preprocessors.nlp import \ DocumentGroundedDialogRetrievalPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class DocumentGroundedDialogRetrievalTest(unittest.TestCase, - DemoCompatibilityCheck): +class DocumentGroundedDialogRetrievalTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.document_grounded_dialog_retrieval diff --git a/tests/pipelines/test_document_segmentation.py b/tests/pipelines/test_document_segmentation.py index 41c490d2..09ce5756 100644 --- a/tests/pipelines/test_document_segmentation.py +++ b/tests/pipelines/test_document_segmentation.py @@ -6,14 +6,13 @@ from typing import Any, Dict from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import test_level logger = get_logger() -class DocumentSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): +class DocumentSegmentationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.document_segmentation @@ -64,10 +63,6 @@ class DocumentSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): for document in documents_list: print(document) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_document_vl_embedding.py b/tests/pipelines/test_document_vl_embedding.py index f8d2d5a3..349547d1 100644 --- a/tests/pipelines/test_document_vl_embedding.py +++ b/tests/pipelines/test_document_vl_embedding.py @@ -10,11 +10,10 @@ from modelscope.models import Model from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class DocumentVLEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck): +class DocumentVLEmbeddingTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/multi-modal_convnext-roberta-base_vldoc-embedding' @@ -51,10 +50,6 @@ class DocumentVLEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck): vldoc_doc_VL_emb_pipeline = pipeline(self.task) self.pipeline_inference(vldoc_doc_VL_emb_pipeline) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_domain_classification.py b/tests/pipelines/test_domain_classification.py index 8e5bfa7f..006daa65 100644 --- a/tests/pipelines/test_domain_classification.py +++ b/tests/pipelines/test_domain_classification.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class DomainClassificationTest(unittest.TestCase, DemoCompatibilityCheck): +class DomainClassificationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.text_classification @@ -36,10 +35,6 @@ class DomainClassificationTest(unittest.TestCase, DemoCompatibilityCheck): pipeline_ins = pipeline(self.task, model=model_id) print(pipeline_ins(input=inputs)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_efficient_diffusion_tuning.py b/tests/pipelines/test_efficient_diffusion_tuning.py index 9dc5e412..e33b2bf2 100644 --- a/tests/pipelines/test_efficient_diffusion_tuning.py +++ b/tests/pipelines/test_efficient_diffusion_tuning.py @@ -5,11 +5,10 @@ from modelscope.models import Model from modelscope.models.multi_modal import EfficientStableDiffusion from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class EfficientDiffusionTuningTest(unittest.TestCase, DemoCompatibilityCheck): +class EfficientDiffusionTuningTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.efficient_diffusion_tuning @@ -28,13 +27,9 @@ class EfficientDiffusionTuningTest(unittest.TestCase, DemoCompatibilityCheck): model = Model.from_pretrained(model_id) self.assertTrue(model.__class__ == EfficientStableDiffusion) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_efficient_diffusion_tuning_lora_demo_compatibility(self): - self.model_id = 'damo/multi-modal_efficient-diffusion-tuning-lora' - self.compatibility_check() - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_efficient_diffusion_tuning_control_lora_run_pipeline(self): + # TODO: to be fixed in the future model_id = 'damo/multi-modal_efficient-diffusion-tuning-control-lora' inputs = { 'prompt': @@ -53,11 +48,6 @@ class EfficientDiffusionTuningTest(unittest.TestCase, DemoCompatibilityCheck): model = Model.from_pretrained(model_id) self.assertTrue(model.__class__ == EfficientStableDiffusion) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_efficient_diffusion_tuning_control_lora_demo_compatibility(self): - self.model_id = 'damo/multi-modal_efficient-diffusion-tuning-control-lora' - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_extractive_summarization.py b/tests/pipelines/test_extractive_summarization.py index 26ac508c..a7f12d14 100644 --- a/tests/pipelines/test_extractive_summarization.py +++ b/tests/pipelines/test_extractive_summarization.py @@ -6,14 +6,13 @@ from typing import Any, Dict from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import test_level logger = get_logger() -class ExtractiveSummarizationTest(unittest.TestCase, DemoCompatibilityCheck): +class ExtractiveSummarizationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.extractive_summarization @@ -46,10 +45,6 @@ class ExtractiveSummarizationTest(unittest.TestCase, DemoCompatibilityCheck): model_id=self.ponet_topic_model_id, documents=self.sentences) print(result[OutputKeys.TEXT]) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_face_detection.py b/tests/pipelines/test_face_detection.py index 7e35cdbb..86e0f702 100644 --- a/tests/pipelines/test_face_detection.py +++ b/tests/pipelines/test_face_detection.py @@ -8,11 +8,10 @@ from modelscope.msdatasets import MsDataset from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import draw_face_detection_result -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class FaceDetectionTest(unittest.TestCase, DemoCompatibilityCheck): +class FaceDetectionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.face_detection @@ -42,10 +41,6 @@ class FaceDetectionTest(unittest.TestCase, DemoCompatibilityCheck): result = face_detection(img_path) self.show_result(img_path, result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_face_image_generation.py b/tests/pipelines/test_face_image_generation.py index 21d8e835..fbd7e3b5 100644 --- a/tests/pipelines/test_face_image_generation.py +++ b/tests/pipelines/test_face_image_generation.py @@ -8,11 +8,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class FaceGenerationTest(unittest.TestCase, DemoCompatibilityCheck): +class FaceGenerationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.face_image_generation @@ -39,10 +38,6 @@ class FaceGenerationTest(unittest.TestCase, DemoCompatibilityCheck): face_generation = pipeline(self.task) self.pipeline_inference(face_generation, seed) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_face_recognition.py b/tests/pipelines/test_face_recognition.py index d3451f5d..7b84590c 100644 --- a/tests/pipelines/test_face_recognition.py +++ b/tests/pipelines/test_face_recognition.py @@ -6,11 +6,10 @@ import numpy as np from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class FaceRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): +class FaceRecognitionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.face_recognition @@ -28,10 +27,6 @@ class FaceRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): sim = np.dot(emb1[0], emb2[0]) print(f'Cos similarity={sim:.3f}, img1:{img1} img2:{img2}') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_face_recognition_onnx_fm.py b/tests/pipelines/test_face_recognition_onnx_fm.py index 8478b3bf..b60bec93 100644 --- a/tests/pipelines/test_face_recognition_onnx_fm.py +++ b/tests/pipelines/test_face_recognition_onnx_fm.py @@ -6,11 +6,10 @@ import numpy as np from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class FmFaceRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): +class FmFaceRecognitionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.face_recognition @@ -31,10 +30,6 @@ class FmFaceRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): sim = np.dot(emb1[0], emb2[0]) print(f'Cos similarity={sim:.3f}, img1:{img1} img2:{img2}') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_face_recognition_onnx_ir.py b/tests/pipelines/test_face_recognition_onnx_ir.py index c45042be..a7cf008c 100644 --- a/tests/pipelines/test_face_recognition_onnx_ir.py +++ b/tests/pipelines/test_face_recognition_onnx_ir.py @@ -6,11 +6,10 @@ import numpy as np from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class IrFaceRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): +class IrFaceRecognitionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.face_recognition @@ -31,10 +30,6 @@ class IrFaceRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): sim = np.dot(emb1[0], emb2[0]) print(f'Cos similarity={sim:.3f}, img1:{img1} img2:{img2}') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_face_recognition_ood.py b/tests/pipelines/test_face_recognition_ood.py index 8a6fb444..68cf1f0b 100644 --- a/tests/pipelines/test_face_recognition_ood.py +++ b/tests/pipelines/test_face_recognition_ood.py @@ -6,11 +6,10 @@ import numpy as np from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class FaceRecognitionOodTest(unittest.TestCase, DemoCompatibilityCheck): +class FaceRecognitionOodTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.face_recognition @@ -39,10 +38,6 @@ class FaceRecognitionOodTest(unittest.TestCase, DemoCompatibilityCheck): print(f'Cos similarity={sim:.3f}, img1:{img1} img2:{img2}') print(f'OOD score: img1:{score1:.3f} img2:{score2:.3f}') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_face_reconstruction.py b/tests/pipelines/test_face_reconstruction.py index c1de97c0..06950487 100644 --- a/tests/pipelines/test_face_reconstruction.py +++ b/tests/pipelines/test_face_reconstruction.py @@ -14,13 +14,12 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level sys.path.append('.') -class FaceReconstructionTest(unittest.TestCase, DemoCompatibilityCheck): +class FaceReconstructionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.face_reconstruction @@ -68,10 +67,6 @@ class FaceReconstructionTest(unittest.TestCase, DemoCompatibilityCheck): model_revision='v2.0.0-HRN') self.pipeline_inference(face_reconstruction, self.test_image) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_faq_question_answering.py b/tests/pipelines/test_faq_question_answering.py index 31680095..89f95162 100644 --- a/tests/pipelines/test_faq_question_answering.py +++ b/tests/pipelines/test_faq_question_answering.py @@ -12,11 +12,10 @@ from modelscope.pipelines.nlp import FaqQuestionAnsweringPipeline from modelscope.preprocessors import \ FaqQuestionAnsweringTransformersPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class FaqQuestionAnsweringTest(unittest.TestCase, DemoCompatibilityCheck): +class FaqQuestionAnsweringTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.faq_question_answering @@ -103,10 +102,6 @@ class FaqQuestionAnsweringTest(unittest.TestCase, DemoCompatibilityCheck): ['今天星期六', '明天星期几明天星期几']) print(np.shape(sentence_vec)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_feature_extraction.py b/tests/pipelines/test_feature_extraction.py index da6be1c0..8e365eab 100644 --- a/tests/pipelines/test_feature_extraction.py +++ b/tests/pipelines/test_feature_extraction.py @@ -11,12 +11,10 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import FeatureExtractionPipeline from modelscope.preprocessors import FillMaskTransformersPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class FeatureExtractionTaskModelTest(unittest.TestCase, - DemoCompatibilityCheck): +class FeatureExtractionTaskModelTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.feature_extraction diff --git a/tests/pipelines/test_fid_dialogue.py b/tests/pipelines/test_fid_dialogue.py index 600501ef..96d7c919 100644 --- a/tests/pipelines/test_fid_dialogue.py +++ b/tests/pipelines/test_fid_dialogue.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class FidDialogueTest(unittest.TestCase, DemoCompatibilityCheck): +class FidDialogueTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.fid_dialogue @@ -67,10 +66,6 @@ class FidDialogueTest(unittest.TestCase, DemoCompatibilityCheck): result = pipeline_ins(self.input, **self.kwargs) print(result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_fill_mask.py b/tests/pipelines/test_fill_mask.py index 0e427464..450ada15 100644 --- a/tests/pipelines/test_fill_mask.py +++ b/tests/pipelines/test_fill_mask.py @@ -10,12 +10,11 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import FillMaskPipeline from modelscope.preprocessors import FillMaskTransformersPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.regress_test_utils import IgnoreKeyFn, MsRegressTool from modelscope.utils.test_utils import test_level -class FillMaskTest(unittest.TestCase, DemoCompatibilityCheck): +class FillMaskTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.fill_mask @@ -134,8 +133,9 @@ class FillMaskTest(unittest.TestCase, DemoCompatibilityCheck): f'\nori_text: {ori_text}\ninput: {test_input}\npipeline: ' f'{pipeline_ins(test_input)}\n') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_run_with_model_name(self): + # TODO: to be fixed in the future # veco pipeline_ins = pipeline(task=Tasks.fill_mask, model=self.model_id_veco) for language in ['zh', 'en']: @@ -176,10 +176,6 @@ class FillMaskTest(unittest.TestCase, DemoCompatibilityCheck): print(f'\nori_text: {ori_text}\ninput: {test_input}\npipeline: ' f'{pipeline_ins(test_input)}\n') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_general_image_classification.py b/tests/pipelines/test_general_image_classification.py index 978c474a..df036fa1 100644 --- a/tests/pipelines/test_general_image_classification.py +++ b/tests/pipelines/test_general_image_classification.py @@ -4,13 +4,11 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.regress_test_utils import MsRegressTool from modelscope.utils.test_utils import test_level -class GeneralImageClassificationTest(unittest.TestCase, - DemoCompatibilityCheck): +class GeneralImageClassificationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_classification @@ -83,10 +81,6 @@ class GeneralImageClassificationTest(unittest.TestCase, result = general_image_classification('data/test/images/bird.JPEG') print(result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_general_recognition.py b/tests/pipelines/test_general_recognition.py index ba713bbe..873aaa02 100644 --- a/tests/pipelines/test_general_recognition.py +++ b/tests/pipelines/test_general_recognition.py @@ -4,11 +4,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class GeneralRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): +class GeneralRecognitionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.general_recognition @@ -22,10 +21,6 @@ class GeneralRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): result = general_recognition('data/test/images/dogs.jpg') print(result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_generative_multi_modal_embedding.py b/tests/pipelines/test_generative_multi_modal_embedding.py index 18b96f65..3a853725 100644 --- a/tests/pipelines/test_generative_multi_modal_embedding.py +++ b/tests/pipelines/test_generative_multi_modal_embedding.py @@ -5,11 +5,10 @@ import unittest from modelscope.models import Model from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class GEMMMultiModalEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck): +class GEMMMultiModalEmbeddingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.generative_multi_modal_embedding @@ -68,10 +67,6 @@ class GEMMMultiModalEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck): output = generative_multi_modal_embedding_pipeline(test_input) print(output) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_gridvlp_classification.py b/tests/pipelines/test_gridvlp_classification.py index 18c6c582..7479d0fa 100644 --- a/tests/pipelines/test_gridvlp_classification.py +++ b/tests/pipelines/test_gridvlp_classification.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines.multi_modal.gridvlp_pipeline import ( GridVlpClassificationPipeline, GridVlpEmbeddingPipeline) -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class GridVlpClassificationTest(unittest.TestCase, DemoCompatibilityCheck): +class GridVlpClassificationTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'rgtjf1/multi-modal_gridvlp_classification_chinese-base-ecom-cate' @@ -62,10 +61,6 @@ class GridVlpClassificationTest(unittest.TestCase, DemoCompatibilityCheck): print(f'text: {self.text}\nimage: {self.image}\n' f'outputs shape: {outputs.shape}') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_hand_detection.py b/tests/pipelines/test_hand_detection.py index 60366287..9ea192a1 100644 --- a/tests/pipelines/test_hand_detection.py +++ b/tests/pipelines/test_hand_detection.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): +class ObjectDetectionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.domain_specific_object_detection diff --git a/tests/pipelines/test_hicossl_video_embedding.py b/tests/pipelines/test_hicossl_video_embedding.py index 8a7de1fa..a367457f 100644 --- a/tests/pipelines/test_hicossl_video_embedding.py +++ b/tests/pipelines/test_hicossl_video_embedding.py @@ -4,11 +4,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class HICOSSLVideoEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck): +class HICOSSLVideoEmbeddingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_embedding @@ -23,10 +22,6 @@ class HICOSSLVideoEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck): print(f'video embedding output: {result}.') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_hitea_tasks.py b/tests/pipelines/test_hitea_tasks.py index 50efdfbd..60cd206d 100644 --- a/tests/pipelines/test_hitea_tasks.py +++ b/tests/pipelines/test_hitea_tasks.py @@ -5,11 +5,10 @@ from modelscope.models import Model from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class HiTeATasksTest(unittest.TestCase, DemoCompatibilityCheck): +class HiTeATasksTest(unittest.TestCase): @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_run_with_video_captioning_with_model(self): @@ -55,10 +54,6 @@ class HiTeATasksTest(unittest.TestCase, DemoCompatibilityCheck): result = pipeline_vqa(input) print(result) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_body_reshaping.py b/tests/pipelines/test_image_body_reshaping.py index e1955e94..5a0ec0e7 100644 --- a/tests/pipelines/test_image_body_reshaping.py +++ b/tests/pipelines/test_image_body_reshaping.py @@ -9,11 +9,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageBodyReshapingTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageBodyReshapingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_body_reshaping @@ -49,10 +48,6 @@ class ImageBodyReshapingTest(unittest.TestCase, DemoCompatibilityCheck): image_body_reshaping = pipeline(Tasks.image_body_reshaping) self.pipeline_inference(image_body_reshaping, self.test_image) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_color_enhance.py b/tests/pipelines/test_image_color_enhance.py index 7c3ae8c0..5e222776 100644 --- a/tests/pipelines/test_image_color_enhance.py +++ b/tests/pipelines/test_image_color_enhance.py @@ -8,11 +8,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageColorEnhanceTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageColorEnhanceTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_csrnet_image-color-enhance-models' @@ -37,10 +36,6 @@ class ImageColorEnhanceTest(unittest.TestCase, DemoCompatibilityCheck): self.pipeline_inference(img_color_enhance, 'data/test/images/image_color_enhance.png') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_colorization.py b/tests/pipelines/test_image_colorization.py index 547fce89..15ea314a 100644 --- a/tests/pipelines/test_image_colorization.py +++ b/tests/pipelines/test_image_colorization.py @@ -8,11 +8,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageColorizationTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageColorizationTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_unet_image-colorization' @@ -37,10 +36,6 @@ class ImageColorizationTest(unittest.TestCase, DemoCompatibilityCheck): image_colorization = pipeline(Tasks.image_colorization) self.pipeline_inference(image_colorization, self.test_image) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_debanding.py b/tests/pipelines/test_image_debanding.py index 105d1f45..da784596 100644 --- a/tests/pipelines/test_image_debanding.py +++ b/tests/pipelines/test_image_debanding.py @@ -8,11 +8,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageColorEnhanceTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageColorEnhanceTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_rrdb_image-debanding' @@ -36,10 +35,6 @@ class ImageColorEnhanceTest(unittest.TestCase, DemoCompatibilityCheck): self.pipeline_inference(img_debanding, 'data/test/images/image_debanding.png') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_deblur.py b/tests/pipelines/test_image_deblur.py index fc9d0101..529ae96c 100644 --- a/tests/pipelines/test_image_deblur.py +++ b/tests/pipelines/test_image_deblur.py @@ -8,11 +8,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.cv import ImageDeblurPipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageDenoiseTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageDenoiseTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_deblurring @@ -56,10 +55,6 @@ class ImageDenoiseTest(unittest.TestCase, DemoCompatibilityCheck): h, w = deblur_img.shape[:2] print('pipeline: the shape of output_img is {}x{}'.format(h, w)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_defrcn_fewshot.py b/tests/pipelines/test_image_defrcn_fewshot.py index d2ecde13..1771d7b8 100644 --- a/tests/pipelines/test_image_defrcn_fewshot.py +++ b/tests/pipelines/test_image_defrcn_fewshot.py @@ -8,14 +8,13 @@ from modelscope.models import Model from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import test_level logger = get_logger() -class ImageDefrcnFewShotTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageDefrcnFewShotTest(unittest.TestCase): def setUp(self) -> None: logger.info('start install detectron2-0.3') @@ -58,10 +57,6 @@ class ImageDefrcnFewShotTest(unittest.TestCase, DemoCompatibilityCheck): self.task, model=cache_path, model_revision=self.revision) print(pipeline_defrcn(input=self.image)[OutputKeys.LABELS]) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_denoise.py b/tests/pipelines/test_image_denoise.py index d95dd343..891e703e 100644 --- a/tests/pipelines/test_image_denoise.py +++ b/tests/pipelines/test_image_denoise.py @@ -8,11 +8,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.cv import ImageDenoisePipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageDenoiseTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageDenoiseTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_denoising @@ -56,10 +55,6 @@ class ImageDenoiseTest(unittest.TestCase, DemoCompatibilityCheck): h, w = denoise_img.shape[:2] print('pipeline: the shape of output_img is {}x{}'.format(h, w)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_depth_estimation.py b/tests/pipelines/test_image_depth_estimation.py index 6ec16a64..7f9b3bb9 100644 --- a/tests/pipelines/test_image_depth_estimation.py +++ b/tests/pipelines/test_image_depth_estimation.py @@ -9,11 +9,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import depth_to_color -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageDepthEstimationTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageDepthEstimationTest(unittest.TestCase): def setUp(self) -> None: self.task = 'image-depth-estimation' diff --git a/tests/pipelines/test_image_depth_estimation_bts.py b/tests/pipelines/test_image_depth_estimation_bts.py index bda7a41f..e952da30 100644 --- a/tests/pipelines/test_image_depth_estimation_bts.py +++ b/tests/pipelines/test_image_depth_estimation_bts.py @@ -8,11 +8,10 @@ from modelscope.models import Model from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageDepthEstimationBtsTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageDepthEstimationBtsTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_depth_estimation @@ -45,10 +44,6 @@ class ImageDepthEstimationBtsTest(unittest.TestCase, DemoCompatibilityCheck): cv2.imwrite('result_snapshot.jpg', depth_vis) print('Test run with snapshot ok.') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_driving_perception.py b/tests/pipelines/test_image_driving_perception.py index 2f28b7d3..a6ad902d 100644 --- a/tests/pipelines/test_image_driving_perception.py +++ b/tests/pipelines/test_image_driving_perception.py @@ -17,11 +17,10 @@ from modelscope.preprocessors.image import LoadImage from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import \ show_image_driving_perception_result -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageDrivingPerceptionTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageDrivingPerceptionTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_yolopv2_image-driving-perception_bdd100k' @@ -59,10 +58,6 @@ class ImageDrivingPerceptionTest(unittest.TestCase, DemoCompatibilityCheck): self.pipeline_inference(image_driving_perception_pipeline, self.img_path) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_face_fusion.py b/tests/pipelines/test_image_face_fusion.py index fde15edf..54d2c3a4 100644 --- a/tests/pipelines/test_image_face_fusion.py +++ b/tests/pipelines/test_image_face_fusion.py @@ -7,11 +7,10 @@ from modelscope.hub.snapshot_download import snapshot_download from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageFaceFusionTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageFaceFusionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_face_fusion @@ -50,10 +49,6 @@ class ImageFaceFusionTest(unittest.TestCase, DemoCompatibilityCheck): cv2.imwrite('result_facefusion.png', result[OutputKeys.OUTPUT_IMG]) print('facefusion.test_run_modelhub_default_model done') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_human_parsing.py b/tests/pipelines/test_image_human_parsing.py index 77d75862..f9263ea8 100644 --- a/tests/pipelines/test_image_human_parsing.py +++ b/tests/pipelines/test_image_human_parsing.py @@ -5,11 +5,10 @@ from modelscope.models import Model from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageHumanParsingTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageHumanParsingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_segmentation @@ -39,10 +38,6 @@ class ImageHumanParsingTest(unittest.TestCase, DemoCompatibilityCheck): task=Tasks.image_segmentation, model=model, preprocessor=None) print(pipeline_parsing(input=self.image_multiple)[OutputKeys.LABELS]) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_inpainting_sdv2.py b/tests/pipelines/test_image_inpainting_sdv2.py index 81002ce8..b21ac69d 100644 --- a/tests/pipelines/test_image_inpainting_sdv2.py +++ b/tests/pipelines/test_image_inpainting_sdv2.py @@ -10,11 +10,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.cv import ImageInpaintingSDV2Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageInpaintingSDV2Test(unittest.TestCase, DemoCompatibilityCheck): +class ImageInpaintingSDV2Test(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_inpainting @@ -50,10 +49,6 @@ class ImageInpaintingSDV2Test(unittest.TestCase, DemoCompatibilityCheck): print( 'pipeline: the output image path is {}'.format(output_image_path)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_instance_segmentation.py b/tests/pipelines/test_image_instance_segmentation.py index 2ba0724a..c305a7c0 100644 --- a/tests/pipelines/test_image_instance_segmentation.py +++ b/tests/pipelines/test_image_instance_segmentation.py @@ -12,11 +12,10 @@ from modelscope.pipelines.cv import ImageInstanceSegmentationPipeline from modelscope.preprocessors import build_preprocessor from modelscope.utils.config import Config from modelscope.utils.constant import Fields, ModelFile, Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageInstanceSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageInstanceSegmentationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_segmentation @@ -61,10 +60,6 @@ class ImageInstanceSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): print(f'pipeline1:{pipeline1(input=self.image)[OutputKeys.LABELS]}') print(f'pipeline2: {pipeline2(input=self.image)[OutputKeys.LABELS]}') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_layout_estimation.py b/tests/pipelines/test_image_layout_estimation.py index b312e8c2..4c93fa30 100644 --- a/tests/pipelines/test_image_layout_estimation.py +++ b/tests/pipelines/test_image_layout_estimation.py @@ -7,11 +7,10 @@ import cv2 from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageLayoutEstimationTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageLayoutEstimationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.indoor_layout_estimation diff --git a/tests/pipelines/test_image_matching.py b/tests/pipelines/test_image_matching.py index 55fd56df..6007ea31 100644 --- a/tests/pipelines/test_image_matching.py +++ b/tests/pipelines/test_image_matching.py @@ -11,11 +11,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import match_pair_visualization -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageMatchingTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageMatchingTest(unittest.TestCase): def setUp(self) -> None: self.task = 'image-matching' diff --git a/tests/pipelines/test_image_matting.py b/tests/pipelines/test_image_matting.py index a3edb705..d6d87a0c 100644 --- a/tests/pipelines/test_image_matting.py +++ b/tests/pipelines/test_image_matting.py @@ -8,11 +8,10 @@ from modelscope.msdatasets import MsDataset from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import ModelFile, Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageMattingTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageMattingTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_unet_image-matting' @@ -61,10 +60,6 @@ class ImageMattingTest(unittest.TestCase, DemoCompatibilityCheck): f'Output written to dir: {osp.dirname(osp.abspath("result_0.png"))}' ) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_mvs_depth_estimation.py b/tests/pipelines/test_image_mvs_depth_estimation.py index a7e327e3..b158623b 100644 --- a/tests/pipelines/test_image_mvs_depth_estimation.py +++ b/tests/pipelines/test_image_mvs_depth_estimation.py @@ -6,11 +6,10 @@ from modelscope.hub.snapshot_download import snapshot_download from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageMVSDepthEstimationTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageMVSDepthEstimationTest(unittest.TestCase): def setUp(self) -> None: self.task = 'image-multi-view-depth-estimation' diff --git a/tests/pipelines/test_image_open_vocabulary_detection.py b/tests/pipelines/test_image_open_vocabulary_detection.py index 52dc1d11..923e1efe 100644 --- a/tests/pipelines/test_image_open_vocabulary_detection.py +++ b/tests/pipelines/test_image_open_vocabulary_detection.py @@ -10,15 +10,13 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import draw_box -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import test_level logger = get_logger() -class ImageOpenVocabularyDetectionTest(unittest.TestCase, - DemoCompatibilityCheck): +class ImageOpenVocabularyDetectionTest(unittest.TestCase): def setUp(self) -> None: os.system( @@ -74,10 +72,6 @@ class ImageOpenVocabularyDetectionTest(unittest.TestCase, cv2.imwrite('result_snapshot.jpg', image) print('Test run with snapshot ok.') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_panoptic_segmentation.py b/tests/pipelines/test_image_panoptic_segmentation.py index ccd9eafd..38c66055 100644 --- a/tests/pipelines/test_image_panoptic_segmentation.py +++ b/tests/pipelines/test_image_panoptic_segmentation.py @@ -9,11 +9,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import panoptic_seg_masks_to_image -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImagePanopticSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): +class ImagePanopticSegmentationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_segmentation @@ -40,10 +39,6 @@ class ImagePanopticSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): cv2.imwrite('result.jpg', draw_img) print('print test_image_panoptic_segmentation from PIL return success') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_portrait_enhancement.py b/tests/pipelines/test_image_portrait_enhancement.py index f0814c07..43978fd2 100644 --- a/tests/pipelines/test_image_portrait_enhancement.py +++ b/tests/pipelines/test_image_portrait_enhancement.py @@ -9,11 +9,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImagePortraitEnhancementTest(unittest.TestCase, DemoCompatibilityCheck): +class ImagePortraitEnhancementTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_portrait_enhancement @@ -46,10 +45,6 @@ class ImagePortraitEnhancementTest(unittest.TestCase, DemoCompatibilityCheck): face_enhancement = pipeline(Tasks.image_portrait_enhancement) self.pipeline_inference(face_enhancement, self.test_image) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_quality_assessment_degradation.py b/tests/pipelines/test_image_quality_assessment_degradation.py index cb0f24c7..1acf8163 100644 --- a/tests/pipelines/test_image_quality_assessment_degradation.py +++ b/tests/pipelines/test_image_quality_assessment_degradation.py @@ -9,14 +9,12 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.cv import ImageQualityAssessmentDegradationPipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level sys.path.insert(0, '.') -class ImageQualityAssessmentDegradationTest(unittest.TestCase, - DemoCompatibilityCheck): +class ImageQualityAssessmentDegradationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_quality_assessment_degradation @@ -54,10 +52,6 @@ class ImageQualityAssessmentDegradationTest(unittest.TestCase, out_path = pipeline_ins(input=self.test_img)[OutputKeys.SCORES] print('pipeline: the out_path is {}'.format(out_path)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_quality_assessment_man.py b/tests/pipelines/test_image_quality_assessment_man.py index 2668d45d..f36f8b3c 100644 --- a/tests/pipelines/test_image_quality_assessment_man.py +++ b/tests/pipelines/test_image_quality_assessment_man.py @@ -7,11 +7,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.cv import ImageQualityAssessmentMANPipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageQualityAssessmentMANTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageQualityAssessmentMANTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_quality_assessment_mos @@ -47,10 +46,6 @@ class ImageQualityAssessmentMANTest(unittest.TestCase, DemoCompatibilityCheck): out_path = pipeline_ins(input=self.test_img)[OutputKeys.SCORE] print('pipeline: the out_path is {}'.format(out_path)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_quality_assessment_mos.py b/tests/pipelines/test_image_quality_assessment_mos.py index 608be8f8..3ca26b0a 100644 --- a/tests/pipelines/test_image_quality_assessment_mos.py +++ b/tests/pipelines/test_image_quality_assessment_mos.py @@ -7,11 +7,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.cv import ImageQualityAssessmentMosPipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageQualityAssessmentMosTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageQualityAssessmentMosTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_quality_assessment_mos @@ -47,10 +46,6 @@ class ImageQualityAssessmentMosTest(unittest.TestCase, DemoCompatibilityCheck): out_path = pipeline_ins(input=self.test_img)[OutputKeys.SCORE] print('pipeline: the out_path is {}'.format(out_path)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_reid_person.py b/tests/pipelines/test_image_reid_person.py index 310cdd66..e107d5ee 100644 --- a/tests/pipelines/test_image_reid_person.py +++ b/tests/pipelines/test_image_reid_person.py @@ -6,11 +6,10 @@ from PIL import Image from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageReidPersonTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageReidPersonTest(unittest.TestCase): def setUp(self) -> None: self.input_location = 'data/test/images/image_reid_person.jpg' @@ -50,10 +49,6 @@ class ImageReidPersonTest(unittest.TestCase, DemoCompatibilityCheck): ) print(f'The img embedding is: {result[OutputKeys.IMG_EMBEDDING]}') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_restoration.py b/tests/pipelines/test_image_restoration.py index baffa7d5..b9c600b2 100644 --- a/tests/pipelines/test_image_restoration.py +++ b/tests/pipelines/test_image_restoration.py @@ -4,11 +4,10 @@ import unittest from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageRestorationTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageRestorationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_demoireing @@ -24,10 +23,6 @@ class ImageRestorationTest(unittest.TestCase, DemoCompatibilityCheck): Image.fromarray(result[OutputKeys.OUTPUT_IMG]).save(input_location + '_demoire.jpg') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_semantic_segmentation.py b/tests/pipelines/test_image_semantic_segmentation.py index 2e8d7522..5bc89bd1 100644 --- a/tests/pipelines/test_image_semantic_segmentation.py +++ b/tests/pipelines/test_image_semantic_segmentation.py @@ -9,11 +9,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import semantic_seg_masks_to_image -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageSemanticSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageSemanticSegmentationTest(unittest.TestCase): def setUp(self) -> None: self.task = 'image-segmentation' @@ -54,10 +53,6 @@ class ImageSemanticSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): cv2.imwrite('result.jpg', draw_img) print('test_image_semantic_segmentation_vitadapter_from_PIL DONE') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_structured_model_probing.py b/tests/pipelines/test_image_structured_model_probing.py index 1befcf98..f4d46d92 100644 --- a/tests/pipelines/test_image_structured_model_probing.py +++ b/tests/pipelines/test_image_structured_model_probing.py @@ -4,12 +4,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageStructuredModelProbingTest(unittest.TestCase, - DemoCompatibilityCheck): +class ImageStructuredModelProbingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_classification diff --git a/tests/pipelines/test_image_style_transfer.py b/tests/pipelines/test_image_style_transfer.py index 5f37f204..223ec757 100644 --- a/tests/pipelines/test_image_style_transfer.py +++ b/tests/pipelines/test_image_style_transfer.py @@ -7,11 +7,10 @@ from modelscope.hub.snapshot_download import snapshot_download from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageStyleTransferTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageStyleTransferTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_style_transfer @@ -53,10 +52,6 @@ class ImageStyleTransferTest(unittest.TestCase, DemoCompatibilityCheck): cv2.imwrite('result_styletransfer3.png', result[OutputKeys.OUTPUT_IMG]) print('style_transfer.test_run_modelhub_default_model done') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_image_super_resolution.py b/tests/pipelines/test_image_super_resolution.py index d5cbebe8..45066300 100644 --- a/tests/pipelines/test_image_super_resolution.py +++ b/tests/pipelines/test_image_super_resolution.py @@ -8,11 +8,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageSuperResolutionTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageSuperResolutionTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_rrdb_image-super-resolution' @@ -37,10 +36,6 @@ class ImageSuperResolutionTest(unittest.TestCase, DemoCompatibilityCheck): super_resolution = pipeline(Tasks.image_super_resolution) self.pipeline_inference(super_resolution, self.img) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_interactive_translation_pipeline.py b/tests/pipelines/test_interactive_translation_pipeline.py index b973250a..c240ba70 100644 --- a/tests/pipelines/test_interactive_translation_pipeline.py +++ b/tests/pipelines/test_interactive_translation_pipeline.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class InteractiveTranslationTest(unittest.TestCase, DemoCompatibilityCheck): +class InteractiveTranslationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.translation @@ -28,10 +27,6 @@ class InteractiveTranslationTest(unittest.TestCase, DemoCompatibilityCheck): pipeline_ins = pipeline(self.task, model=model_id) print(pipeline_ins(inputs + '' + prefix)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_inverse_text_processing.py b/tests/pipelines/test_inverse_text_processing.py index dc7fb1e0..a1d5a712 100644 --- a/tests/pipelines/test_inverse_text_processing.py +++ b/tests/pipelines/test_inverse_text_processing.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class InverseTextProcessingTest(unittest.TestCase, DemoCompatibilityCheck): +class InverseTextProcessingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.inverse_text_processing, @@ -61,10 +60,6 @@ class InverseTextProcessingTest(unittest.TestCase, DemoCompatibilityCheck): itn_result = itn_inference_pipline(text_in=lang_text_in) print(itn_result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_key_word_spotting.py b/tests/pipelines/test_key_word_spotting.py index 13f7a308..f22bc845 100644 --- a/tests/pipelines/test_key_word_spotting.py +++ b/tests/pipelines/test_key_word_spotting.py @@ -10,7 +10,6 @@ import soundfile from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import ColorCodes, Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import download_and_untar, test_level @@ -27,7 +26,7 @@ NEG_TESTSETS_FILE = 'neg_testsets.tar.gz' NEG_TESTSETS_URL = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/KWS/neg_testsets.tar.gz' -class KeyWordSpottingTest(unittest.TestCase, DemoCompatibilityCheck): +class KeyWordSpottingTest(unittest.TestCase): action_info = { 'test_run_with_wav': { 'checking_item': [OutputKeys.KWS_LIST, 0, 'keyword'], @@ -344,10 +343,6 @@ class KeyWordSpottingTest(unittest.TestCase, DemoCompatibilityCheck): model_id=model_id, audio_in=wav_path, keywords=keywords) logger.info(ColorCodes.YELLOW + str(kws_result) + ColorCodes.END) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_language_guided_video_summarization.py b/tests/pipelines/test_language_guided_video_summarization.py index 0f06d4f2..01d88b55 100755 --- a/tests/pipelines/test_language_guided_video_summarization.py +++ b/tests/pipelines/test_language_guided_video_summarization.py @@ -9,12 +9,10 @@ import torch from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class LanguageGuidedVideoSummarizationTest(unittest.TestCase, - DemoCompatibilityCheck): +class LanguageGuidedVideoSummarizationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.language_guided_video_summarization @@ -40,10 +38,6 @@ class LanguageGuidedVideoSummarizationTest(unittest.TestCase, print(f'video summarization output:\n {result}.') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_language_identification.py b/tests/pipelines/test_language_identification.py index ddd91e69..ccfa1a7d 100644 --- a/tests/pipelines/test_language_identification.py +++ b/tests/pipelines/test_language_identification.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class LanguageIdentificationTest(unittest.TestCase, DemoCompatibilityCheck): +class LanguageIdentificationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.text_classification @@ -22,11 +21,6 @@ class LanguageIdentificationTest(unittest.TestCase, DemoCompatibilityCheck): pipeline_ins = pipeline(self.task, model=self.model_id) print(pipeline_ins(input=inputs)) - @unittest.skipUnless(test_level() >= 0, - 'skip test case in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_license_plate_detection.py b/tests/pipelines/test_license_plate_detection.py index 70cdb820..3c30618d 100644 --- a/tests/pipelines/test_license_plate_detection.py +++ b/tests/pipelines/test_license_plate_detection.py @@ -5,11 +5,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class LicensePlateDectionTest(unittest.TestCase, DemoCompatibilityCheck): +class LicensePlateDectionTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_resnet18_license-plate-detection_damo' @@ -32,10 +31,6 @@ class LicensePlateDectionTest(unittest.TestCase, DemoCompatibilityCheck): license_plate_detection = pipeline(Tasks.license_plate_detection) self.pipeline_inference(license_plate_detection, self.test_image) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_lineless_table_recognition.py b/tests/pipelines/test_lineless_table_recognition.py index 53fde8a1..59e173f4 100644 --- a/tests/pipelines/test_lineless_table_recognition.py +++ b/tests/pipelines/test_lineless_table_recognition.py @@ -8,11 +8,10 @@ import numpy as np from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class TableRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): +class TableRecognitionTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_resnet-transformer_table-structure-recognition_lore' @@ -35,10 +34,6 @@ class TableRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): lineless_table_recognition = pipeline(Tasks.lineless_table_recognition) self.pipeline_inference(lineless_table_recognition, self.test_image) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_live_category.py b/tests/pipelines/test_live_category.py index 391ed283..88b94b69 100644 --- a/tests/pipelines/test_live_category.py +++ b/tests/pipelines/test_live_category.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class LiveCategoryTest(unittest.TestCase, DemoCompatibilityCheck): +class LiveCategoryTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.live_category @@ -21,10 +20,6 @@ class LiveCategoryTest(unittest.TestCase, DemoCompatibilityCheck): print(f'live category output: {result}.') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_mask_face_recognition.py b/tests/pipelines/test_mask_face_recognition.py index 550e80e4..2a7e8ede 100644 --- a/tests/pipelines/test_mask_face_recognition.py +++ b/tests/pipelines/test_mask_face_recognition.py @@ -6,11 +6,10 @@ import numpy as np from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class MaskFaceRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): +class MaskFaceRecognitionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.face_recognition @@ -28,10 +27,6 @@ class MaskFaceRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): sim = np.dot(emb1[0], emb2[0]) print(f'Cos similarity={sim:.3f}, img1:{img1} img2:{img2}') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_maskdino_instance_segmentation.py b/tests/pipelines/test_maskdino_instance_segmentation.py index 14e0887d..88c46de1 100644 --- a/tests/pipelines/test_maskdino_instance_segmentation.py +++ b/tests/pipelines/test_maskdino_instance_segmentation.py @@ -8,12 +8,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.cv import MaskDINOInstanceSegmentationPipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class MaskDINOInstanceSegmentationTest(unittest.TestCase, - DemoCompatibilityCheck): +class MaskDINOInstanceSegmentationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_segmentation @@ -45,10 +43,6 @@ class MaskDINOInstanceSegmentationTest(unittest.TestCase, print(f'pipeline1:{pipeline1(input=self.image)[OutputKeys.LABELS]}') print(f'pipeline2: {pipeline2(input=self.image)[OutputKeys.LABELS]}') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_mglm_text_summarization.py b/tests/pipelines/test_mglm_text_summarization.py index 47abc741..703e9bbe 100644 --- a/tests/pipelines/test_mglm_text_summarization.py +++ b/tests/pipelines/test_mglm_text_summarization.py @@ -6,11 +6,10 @@ from modelscope.models import Model from modelscope.pipelines import pipeline from modelscope.preprocessors import MGLMSummarizationPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class mGLMTest(unittest.TestCase, DemoCompatibilityCheck): +class mGLMTest(unittest.TestCase): def setUp(self) -> None: self.output_dir = 'unittest_output' diff --git a/tests/pipelines/test_mobile_image_super_resolution.py b/tests/pipelines/test_mobile_image_super_resolution.py index 2cc7adf0..a486d244 100644 --- a/tests/pipelines/test_mobile_image_super_resolution.py +++ b/tests/pipelines/test_mobile_image_super_resolution.py @@ -8,12 +8,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class MobileImageSuperResolutionTest(unittest.TestCase, - DemoCompatibilityCheck): +class MobileImageSuperResolutionTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_ecbsr_image-super-resolution_mobile' @@ -38,10 +36,6 @@ class MobileImageSuperResolutionTest(unittest.TestCase, super_resolution = pipeline(Tasks.image_super_resolution) self.pipeline_inference(super_resolution, self.img) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_motion_generation.py b/tests/pipelines/test_motion_generation.py index 7938611c..43903eb8 100644 --- a/tests/pipelines/test_motion_generation.py +++ b/tests/pipelines/test_motion_generation.py @@ -4,11 +4,10 @@ import unittest from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class MDMMotionGenerationTest(unittest.TestCase, DemoCompatibilityCheck): +class MDMMotionGenerationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.motion_generation @@ -23,10 +22,6 @@ class MDMMotionGenerationTest(unittest.TestCase, DemoCompatibilityCheck): result[OutputKeys.KEYPOINTS].shape) print('motion generation video file:', result[OutputKeys.OUTPUT_VIDEO]) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_movie_scene_segmentation.py b/tests/pipelines/test_movie_scene_segmentation.py index 0ac8b716..c6498a6c 100644 --- a/tests/pipelines/test_movie_scene_segmentation.py +++ b/tests/pipelines/test_movie_scene_segmentation.py @@ -10,11 +10,10 @@ from modelscope.pipelines import pipeline from modelscope.trainers import build_trainer from modelscope.utils.config import Config, ConfigDict from modelscope.utils.constant import ModelFile, Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class MovieSceneSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): +class MovieSceneSegmentationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.movie_scene_segmentation @@ -123,10 +122,6 @@ class MovieSceneSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): else: raise ValueError('process error') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_mplug_tasks.py b/tests/pipelines/test_mplug_tasks.py index 21439ce2..cff998b4 100644 --- a/tests/pipelines/test_mplug_tasks.py +++ b/tests/pipelines/test_mplug_tasks.py @@ -7,11 +7,10 @@ from modelscope.models import Model from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class MplugTasksTest(unittest.TestCase, DemoCompatibilityCheck): +class MplugTasksTest(unittest.TestCase): @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_run_with_image_captioning_with_model(self): @@ -95,10 +94,6 @@ class MplugTasksTest(unittest.TestCase, DemoCompatibilityCheck): result = pipeline_vqa(input) print(result) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_msrresnetlite_video_super_resolution.py b/tests/pipelines/test_msrresnetlite_video_super_resolution.py index d79e9702..d44cbd34 100644 --- a/tests/pipelines/test_msrresnetlite_video_super_resolution.py +++ b/tests/pipelines/test_msrresnetlite_video_super_resolution.py @@ -7,11 +7,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.cv import VideoSuperResolutionPipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class MSRResNetLiteVSRTest(unittest.TestCase, DemoCompatibilityCheck): +class MSRResNetLiteVSRTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_super_resolution @@ -50,10 +49,6 @@ class MSRResNetLiteVSRTest(unittest.TestCase, DemoCompatibilityCheck): input=self.test_video)[OutputKeys.OUTPUT_VIDEO] print('pipeline: the output video path is {}'.format(out_video_path)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_multi_modal_embedding.py b/tests/pipelines/test_multi_modal_embedding.py index 7eddc690..486adc94 100644 --- a/tests/pipelines/test_multi_modal_embedding.py +++ b/tests/pipelines/test_multi_modal_embedding.py @@ -8,11 +8,10 @@ from modelscope.models import Model from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class MultiModalEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck): +class MultiModalEmbeddingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.multi_modal_embedding @@ -54,10 +53,6 @@ class MultiModalEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck): print('l2-norm: {}'.format(torch.norm(text_embedding, dim=-1).item())) # should be 1.0 - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_multilingual_named_entity_recognition.py b/tests/pipelines/test_multilingual_named_entity_recognition.py index ec134023..a31adf1f 100644 --- a/tests/pipelines/test_multilingual_named_entity_recognition.py +++ b/tests/pipelines/test_multilingual_named_entity_recognition.py @@ -8,12 +8,10 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import NamedEntityRecognitionPipeline from modelscope.preprocessors import NERPreprocessorThai, NERPreprocessorViet from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class MultilingualNamedEntityRecognitionTest(unittest.TestCase, - DemoCompatibilityCheck): +class MultilingualNamedEntityRecognitionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.named_entity_recognition @@ -123,10 +121,6 @@ class MultilingualNamedEntityRecognitionTest(unittest.TestCase, self.viet_sentence[5:] ])) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_multilingual_word_segmentation.py b/tests/pipelines/test_multilingual_word_segmentation.py index f10e6d98..878af0d3 100644 --- a/tests/pipelines/test_multilingual_word_segmentation.py +++ b/tests/pipelines/test_multilingual_word_segmentation.py @@ -8,12 +8,11 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import WordSegmentationThaiPipeline from modelscope.preprocessors import WordSegmentationPreprocessorThai from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.regress_test_utils import MsRegressTool from modelscope.utils.test_utils import test_level -class WordSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): +class WordSegmentationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.word_segmentation @@ -65,10 +64,6 @@ class WordSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): pipeline_ins( input=[self.sentence, self.sentence[:10], self.sentence[6:]])) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_named_entity_recognition.py b/tests/pipelines/test_named_entity_recognition.py index 175e9261..8b7424f4 100644 --- a/tests/pipelines/test_named_entity_recognition.py +++ b/tests/pipelines/test_named_entity_recognition.py @@ -10,11 +10,10 @@ from modelscope.pipelines.nlp import NamedEntityRecognitionPipeline from modelscope.preprocessors import \ TokenClassificationTransformersPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class NamedEntityRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): +class NamedEntityRecognitionTest(unittest.TestCase): language_examples = { 'zh': '新华社北京二月十一日电（记者唐虹）', @@ -470,10 +469,6 @@ class NamedEntityRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): model_id) print(pipeline_ins(input=sentence)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_nerf_recon_acc.py b/tests/pipelines/test_nerf_recon_acc.py index 95d879fb..7ca0fa44 100644 --- a/tests/pipelines/test_nerf_recon_acc.py +++ b/tests/pipelines/test_nerf_recon_acc.py @@ -9,11 +9,10 @@ from modelscope.msdatasets import MsDataset from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import DownloadMode, Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class NeRFReconAccTest(unittest.TestCase, DemoCompatibilityCheck): +class NeRFReconAccTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_nerf-3d-reconstruction-accelerate_damo' @@ -63,11 +62,6 @@ class NeRFReconAccTest(unittest.TestCase, DemoCompatibilityCheck): dict(data_dir=self.data_dir, render_dir=self.render_dir)) print('facefusion.test_run_modelhub_default_model done') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - @unittest.skipIf(not torch.cuda.is_available(), 'cuda unittest only') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_nli.py b/tests/pipelines/test_nli.py index a7d2a236..5bbe353b 100644 --- a/tests/pipelines/test_nli.py +++ b/tests/pipelines/test_nli.py @@ -7,12 +7,11 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import TextClassificationPipeline from modelscope.preprocessors import TextClassificationTransformersPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.regress_test_utils import IgnoreKeyFn, MsRegressTool from modelscope.utils.test_utils import test_level -class NLITest(unittest.TestCase, DemoCompatibilityCheck): +class NLITest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.nli @@ -78,10 +77,6 @@ class NLITest(unittest.TestCase, DemoCompatibilityCheck): pipeline_ins = pipeline(task=Tasks.nli) print(pipeline_ins(input=(self.sentence1, self.sentence2))) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_object_detecion_3d.py b/tests/pipelines/test_object_detecion_3d.py index bb0eebda..69b75b39 100644 --- a/tests/pipelines/test_object_detecion_3d.py +++ b/tests/pipelines/test_object_detecion_3d.py @@ -10,11 +10,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ObjectDetection3DTest(unittest.TestCase, DemoCompatibilityCheck): +class ObjectDetection3DTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.object_detection_3d @@ -48,10 +47,6 @@ class ObjectDetection3DTest(unittest.TestCase, DemoCompatibilityCheck): detect = pipeline(self.task) self.pipeline_inference(detect, idx) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_object_detection.py b/tests/pipelines/test_object_detection.py index 7be6b0f3..f06d954b 100644 --- a/tests/pipelines/test_object_detection.py +++ b/tests/pipelines/test_object_detection.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): +class ObjectDetectionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.human_detection @@ -43,12 +42,9 @@ class ObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): result = human_detect(input_location) print(result) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - - @unittest.skip('skip test in current test level: no pipeline implemented') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_image_object_detection_auto_pipeline(self): + # TODO: to be fixed in the future model_id = 'damo/cv_yolox_image-object-detection-auto' test_image = 'data/test/images/auto_demo.jpg' diff --git a/tests/pipelines/test_ocr_detection.py b/tests/pipelines/test_ocr_detection.py index 243e274b..0ed2e59c 100644 --- a/tests/pipelines/test_ocr_detection.py +++ b/tests/pipelines/test_ocr_detection.py @@ -4,11 +4,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class OCRDetectionTest(unittest.TestCase, DemoCompatibilityCheck): +class OCRDetectionTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_resnet18_ocr-detection-line-level_damo' @@ -43,10 +42,6 @@ class OCRDetectionTest(unittest.TestCase, DemoCompatibilityCheck): ocr_detection = pipeline(Tasks.ocr_detection) self.pipeline_inference(ocr_detection, self.test_image) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_ocr_recognition.py b/tests/pipelines/test_ocr_recognition.py index 3953ae63..94ee521f 100644 --- a/tests/pipelines/test_ocr_recognition.py +++ b/tests/pipelines/test_ocr_recognition.py @@ -6,11 +6,10 @@ import PIL from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class OCRRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): +class OCRRecognitionTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_convnextTiny_ocr-recognition-general_damo' @@ -154,10 +153,6 @@ class OCRRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): Tasks.ocr_recognition, model_revision='v2.3.0', device='cpu') self.pipeline_inference(ocr_recognition, self.test_image) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_ofa_tasks.py b/tests/pipelines/test_ofa_tasks.py index df1b5647..55c3ae65 100644 --- a/tests/pipelines/test_ofa_tasks.py +++ b/tests/pipelines/test_ofa_tasks.py @@ -11,11 +11,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import created_boxed_image -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class OfaTasksTest(unittest.TestCase, DemoCompatibilityCheck): +class OfaTasksTest(unittest.TestCase): def setUp(self) -> None: self.output_dir = 'unittest_output' @@ -366,10 +365,6 @@ class OfaTasksTest(unittest.TestCase, DemoCompatibilityCheck): for r in result: print(r[OutputKeys.TEXT]) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_panorama_depth_estimation.py b/tests/pipelines/test_panorama_depth_estimation.py index 99e575e3..23552274 100644 --- a/tests/pipelines/test_panorama_depth_estimation.py +++ b/tests/pipelines/test_panorama_depth_estimation.py @@ -9,11 +9,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import depth_to_color -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class PanoramaDepthEstimationTest(unittest.TestCase, DemoCompatibilityCheck): +class PanoramaDepthEstimationTest(unittest.TestCase): def setUp(self) -> None: self.task = 'panorama-depth-estimation' diff --git a/tests/pipelines/test_pedestrian_attribute_recognition.py b/tests/pipelines/test_pedestrian_attribute_recognition.py index c0ace43c..7d58ce12 100644 --- a/tests/pipelines/test_pedestrian_attribute_recognition.py +++ b/tests/pipelines/test_pedestrian_attribute_recognition.py @@ -8,12 +8,10 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import draw_pedestrian_attribute -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class PedestrianAttributeRecognitionTest(unittest.TestCase, - DemoCompatibilityCheck): +class PedestrianAttributeRecognitionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.pedestrian_attribute_recognition @@ -39,10 +37,6 @@ class PedestrianAttributeRecognitionTest(unittest.TestCase, self.pipeline_inference(pedestrian_attribute_recognition, Image.open(self.test_image)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_person_image_cartoon.py b/tests/pipelines/test_person_image_cartoon.py index 1dfaf519..40d26c86 100644 --- a/tests/pipelines/test_person_image_cartoon.py +++ b/tests/pipelines/test_person_image_cartoon.py @@ -8,11 +8,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ImageCartoonTest(unittest.TestCase, DemoCompatibilityCheck): +class ImageCartoonTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_unet_person-image-cartoon_compound-models' @@ -83,10 +82,6 @@ class ImageCartoonTest(unittest.TestCase, DemoCompatibilityCheck): img_cartoon = pipeline(Tasks.image_portrait_stylization) self.pipeline_inference(img_cartoon, self.test_image) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_pointcloud_sceneflow_estimation.py b/tests/pipelines/test_pointcloud_sceneflow_estimation.py index 34d87f09..4d4bf7f2 100644 --- a/tests/pipelines/test_pointcloud_sceneflow_estimation.py +++ b/tests/pipelines/test_pointcloud_sceneflow_estimation.py @@ -7,12 +7,10 @@ import numpy as np from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class PointCloudSceneFlowEstimationTest(unittest.TestCase, - DemoCompatibilityCheck): +class PointCloudSceneFlowEstimationTest(unittest.TestCase): def setUp(self) -> None: self.task = 'pointcloud-sceneflow-estimation' diff --git a/tests/pipelines/test_product_retrieval_embedding.py b/tests/pipelines/test_product_retrieval_embedding.py index 2483d53a..f194bb7b 100644 --- a/tests/pipelines/test_product_retrieval_embedding.py +++ b/tests/pipelines/test_product_retrieval_embedding.py @@ -8,11 +8,10 @@ from modelscope.models import Model from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ProductRetrievalEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck): +class ProductRetrievalEmbeddingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.product_retrieval_embedding @@ -41,10 +40,6 @@ class ProductRetrievalEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck): result = product_embed(self.img_input)[OutputKeys.IMG_EMBEDDING] print('abs sum value is: {}'.format(np.sum(np.abs(result)))) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_realtime_object_detection.py b/tests/pipelines/test_realtime_object_detection.py index 76e907a7..21450619 100644 --- a/tests/pipelines/test_realtime_object_detection.py +++ b/tests/pipelines/test_realtime_object_detection.py @@ -7,14 +7,13 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import realtime_object_detection_bbox_vis -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import test_level logger = get_logger() -class RealtimeObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): +class RealtimeObjectDetectionTest(unittest.TestCase): def setUp(self) -> None: self.easycv_small_model_id = 'damo/cv_cspnet_image-object-detection_yolox' @@ -46,10 +45,6 @@ class RealtimeObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): else: raise ValueError('process error') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_realtime_video_object_detection.py b/tests/pipelines/test_realtime_video_object_detection.py index 716c9260..d42bda67 100644 --- a/tests/pipelines/test_realtime_video_object_detection.py +++ b/tests/pipelines/test_realtime_video_object_detection.py @@ -9,15 +9,13 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import show_video_object_detection_result -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import test_level logger = get_logger() -class RealtimeVideoObjectDetectionTest(unittest.TestCase, - DemoCompatibilityCheck): +class RealtimeVideoObjectDetectionTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_cspnet_video-object-detection_streamyolo' @@ -53,10 +51,6 @@ class RealtimeVideoObjectDetectionTest(unittest.TestCase, else: raise ValueError('process error') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_referring_video_object_segmentation.py b/tests/pipelines/test_referring_video_object_segmentation.py index 509e9317..2b7de41c 100644 --- a/tests/pipelines/test_referring_video_object_segmentation.py +++ b/tests/pipelines/test_referring_video_object_segmentation.py @@ -3,12 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ReferringVideoObjectSegmentationTest(unittest.TestCase, - DemoCompatibilityCheck): +class ReferringVideoObjectSegmentationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.referring_video_object_segmentation @@ -45,10 +43,6 @@ class ReferringVideoObjectSegmentationTest(unittest.TestCase, else: raise ValueError('process error') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_relation_extraction.py b/tests/pipelines/test_relation_extraction.py index 17ab61fc..44c0b9ad 100644 --- a/tests/pipelines/test_relation_extraction.py +++ b/tests/pipelines/test_relation_extraction.py @@ -8,11 +8,10 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import InformationExtractionPipeline from modelscope.preprocessors import RelationExtractionTransformersPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class RelationExtractionTest(unittest.TestCase, DemoCompatibilityCheck): +class RelationExtractionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.relation_extraction @@ -55,10 +54,6 @@ class RelationExtractionTest(unittest.TestCase, DemoCompatibilityCheck): pipeline_ins = pipeline(task=Tasks.relation_extraction) print(pipeline_ins(input=self.sentence)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_salient_detection.py b/tests/pipelines/test_salient_detection.py index 3101213c..78ae94db 100644 --- a/tests/pipelines/test_salient_detection.py +++ b/tests/pipelines/test_salient_detection.py @@ -4,11 +4,10 @@ import unittest from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class SalientDetectionTest(unittest.TestCase, DemoCompatibilityCheck): +class SalientDetectionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.semantic_segmentation @@ -44,10 +43,6 @@ class SalientDetectionTest(unittest.TestCase, DemoCompatibilityCheck): cv2.imwrite(input_location + '_camouflag.jpg', result[OutputKeys.MASKS]) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_sentence_similarity.py b/tests/pipelines/test_sentence_similarity.py index 233bd3a1..e411158f 100644 --- a/tests/pipelines/test_sentence_similarity.py +++ b/tests/pipelines/test_sentence_similarity.py @@ -11,12 +11,11 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import TextClassificationPipeline from modelscope.preprocessors import TextClassificationTransformersPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.regress_test_utils import IgnoreKeyFn, MsRegressTool from modelscope.utils.test_utils import test_level -class SentenceSimilarityTest(unittest.TestCase, DemoCompatibilityCheck): +class SentenceSimilarityTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.sentence_similarity @@ -110,10 +109,6 @@ class SentenceSimilarityTest(unittest.TestCase, DemoCompatibilityCheck): pipeline_ins = pipeline(task=Tasks.sentence_similarity) print(pipeline_ins(input=(self.sentence1, self.sentence2))) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_sentiment_classification.py b/tests/pipelines/test_sentiment_classification.py index 278f34a8..bb0311ff 100644 --- a/tests/pipelines/test_sentiment_classification.py +++ b/tests/pipelines/test_sentiment_classification.py @@ -9,12 +9,10 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import TextClassificationPipeline from modelscope.preprocessors import TextClassificationTransformersPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class SentimentClassificationTaskModelTest(unittest.TestCase, - DemoCompatibilityCheck): +class SentimentClassificationTaskModelTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.text_classification @@ -63,10 +61,6 @@ class SentimentClassificationTaskModelTest(unittest.TestCase, self.assertTrue( isinstance(pipeline_ins.model, ModelForTextClassification)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_siamese_uie.py b/tests/pipelines/test_siamese_uie.py index 30b38d2e..c5008573 100644 --- a/tests/pipelines/test_siamese_uie.py +++ b/tests/pipelines/test_siamese_uie.py @@ -10,12 +10,11 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import SiameseUiePipeline from modelscope.preprocessors import SiameseUiePreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.regress_test_utils import IgnoreKeyFn, MsRegressTool from modelscope.utils.test_utils import test_level -class ZeroShotClassificationTest(unittest.TestCase, DemoCompatibilityCheck): +class ZeroShotClassificationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.siamese_uie @@ -67,10 +66,6 @@ class ZeroShotClassificationTest(unittest.TestCase, DemoCompatibilityCheck): pipeline_ins = pipeline(task=Tasks.siamese_uie, model_revision='v1.1') print(pipeline_ins(input=self.sentence, schema=self.schema)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_skin_retouching.py b/tests/pipelines/test_skin_retouching.py index db8d89ed..aa1e0c59 100644 --- a/tests/pipelines/test_skin_retouching.py +++ b/tests/pipelines/test_skin_retouching.py @@ -9,11 +9,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class SkinRetouchingTest(unittest.TestCase, DemoCompatibilityCheck): +class SkinRetouchingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.skin_retouching @@ -41,10 +40,6 @@ class SkinRetouchingTest(unittest.TestCase, DemoCompatibilityCheck): skin_retouching = pipeline(Tasks.skin_retouching) self.pipeline_inference(skin_retouching, self.test_image) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_soonet_video_temporal_grounding.py b/tests/pipelines/test_soonet_video_temporal_grounding.py index 21f8027c..4fafeb31 100644 --- a/tests/pipelines/test_soonet_video_temporal_grounding.py +++ b/tests/pipelines/test_soonet_video_temporal_grounding.py @@ -5,12 +5,10 @@ from modelscope.models import Model from modelscope.models.multi_modal.soonet import SOONet from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class SOONetVideoTemporalGroundingTest(unittest.TestCase, - DemoCompatibilityCheck): +class SOONetVideoTemporalGroundingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_temporal_grounding diff --git a/tests/pipelines/test_speaker_verification.py b/tests/pipelines/test_speaker_verification.py index 41335fc8..84a079d2 100644 --- a/tests/pipelines/test_speaker_verification.py +++ b/tests/pipelines/test_speaker_verification.py @@ -7,7 +7,6 @@ from typing import Any, Dict, List, Union from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import test_level @@ -19,7 +18,7 @@ SPEAKER2_A_EN_16K_WAV = 'data/test/audios/speaker2_a_en_16k.wav' SCL_EXAMPLE_WAV = 'data/test/audios/scl_example1.wav' -class SpeakerVerificationTest(unittest.TestCase, DemoCompatibilityCheck): +class SpeakerVerificationTest(unittest.TestCase): ecapatdnn_voxceleb_16k_model_id = 'damo/speech_ecapa-tdnn_sv_en_voxceleb_16k' campplus_voxceleb_16k_model_id = 'damo/speech_campplus_sv_en_voxceleb_16k' rdino_voxceleb_16k_model_id = 'damo/speech_rdino_ecapa_tdnn_sv_en_voxceleb_16k' diff --git a/tests/pipelines/test_speech_separation.py b/tests/pipelines/test_speech_separation.py index 194f84a8..4edb3b43 100644 --- a/tests/pipelines/test_speech_separation.py +++ b/tests/pipelines/test_speech_separation.py @@ -8,13 +8,12 @@ import numpy from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level MIX_SPEECH_FILE = 'data/test/audios/mix_speech.wav' -class SpeechSeparationTest(unittest.TestCase, DemoCompatibilityCheck): +class SpeechSeparationTest(unittest.TestCase): def setUp(self) -> None: pass @@ -32,10 +31,6 @@ class SpeechSeparationTest(unittest.TestCase, DemoCompatibilityCheck): sf.write(save_file, numpy.frombuffer(signal, dtype=numpy.int16), 8000) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_speech_signal_process.py b/tests/pipelines/test_speech_signal_process.py index 2c26cee6..104bf88a 100644 --- a/tests/pipelines/test_speech_signal_process.py +++ b/tests/pipelines/test_speech_signal_process.py @@ -7,7 +7,6 @@ from modelscope.metainfo import Pipelines from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level NEAREND_MIC_FILE = 'data/test/audios/nearend_mic.wav' @@ -24,7 +23,7 @@ NOISE_SPEECH_URL = 'https://modelscope.oss-cn-beijing.aliyuncs.com/' \ 'test/audios/speech_with_noise.wav' -class SpeechSignalProcessTest(unittest.TestCase, DemoCompatibilityCheck): +class SpeechSignalProcessTest(unittest.TestCase): def setUp(self) -> None: pass @@ -150,10 +149,6 @@ class SpeechSignalProcessTest(unittest.TestCase, DemoCompatibilityCheck): w.write(pcm) audio = f.read(block_size) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_table_recognition.py b/tests/pipelines/test_table_recognition.py index 3c6ee74a..6b81fc62 100644 --- a/tests/pipelines/test_table_recognition.py +++ b/tests/pipelines/test_table_recognition.py @@ -5,11 +5,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class TableRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): +class TableRecognitionTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_dla34_table-structure-recognition_cycle-centernet' @@ -32,10 +31,6 @@ class TableRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): table_recognition = pipeline(Tasks.table_recognition) self.pipeline_inference(table_recognition, self.test_image) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_tbs_detection.py b/tests/pipelines/test_tbs_detection.py index ac0dd550..0d5a1283 100644 --- a/tests/pipelines/test_tbs_detection.py +++ b/tests/pipelines/test_tbs_detection.py @@ -2,11 +2,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class ObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): +class ObjectDetectionTest(unittest.TestCase): @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_run_with_model_name(self): diff --git a/tests/pipelines/test_text2text_generation.py b/tests/pipelines/test_text2text_generation.py index 40576a29..d439e033 100644 --- a/tests/pipelines/test_text2text_generation.py +++ b/tests/pipelines/test_text2text_generation.py @@ -8,11 +8,10 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import TextGenerationT5Pipeline from modelscope.preprocessors import TextGenerationT5Preprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class Text2TextGenerationTest(unittest.TestCase, DemoCompatibilityCheck): +class Text2TextGenerationTest(unittest.TestCase): def setUp(self) -> None: self.model_id_generate = 'damo/t5-cn-base-test' @@ -86,10 +85,6 @@ class Text2TextGenerationTest(unittest.TestCase, DemoCompatibilityCheck): pipeline_ins = pipeline(task=Tasks.text2text_generation) print(pipeline_ins(self.input_generate)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_text_classification.py b/tests/pipelines/test_text_classification.py index d07ddbb8..128f86af 100644 --- a/tests/pipelines/test_text_classification.py +++ b/tests/pipelines/test_text_classification.py @@ -7,11 +7,10 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import TextClassificationPipeline from modelscope.preprocessors import TextClassificationTransformersPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class SequenceClassificationTest(unittest.TestCase, DemoCompatibilityCheck): +class SequenceClassificationTest(unittest.TestCase): sentence1 = 'i like this wonderful place' def setUp(self) -> None: @@ -91,10 +90,6 @@ class SequenceClassificationTest(unittest.TestCase, DemoCompatibilityCheck): result = text_classification(dataset) self.printDataset(result) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_text_driven_segmentation.py b/tests/pipelines/test_text_driven_segmentation.py index a67729ff..741787d9 100644 --- a/tests/pipelines/test_text_driven_segmentation.py +++ b/tests/pipelines/test_text_driven_segmentation.py @@ -23,10 +23,6 @@ class TextDrivenSegmentationTest(unittest.TestCase): # result[OutputKeys.MASKS] is segment map result,other keys are not used cv2.imwrite(input_location + '_lseg.jpg', result[OutputKeys.MASKS]) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.test_demo() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_text_error_correction.py b/tests/pipelines/test_text_error_correction.py index 171f3ab2..b4bf5be9 100644 --- a/tests/pipelines/test_text_error_correction.py +++ b/tests/pipelines/test_text_error_correction.py @@ -9,11 +9,10 @@ from modelscope.pipelines.nlp import TextErrorCorrectionPipeline from modelscope.preprocessors import (Preprocessor, TextErrorCorrectionPreprocessor) from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class TextErrorCorrectionTest(unittest.TestCase, DemoCompatibilityCheck): +class TextErrorCorrectionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.text_error_correction @@ -81,10 +80,6 @@ class TextErrorCorrectionTest(unittest.TestCase, DemoCompatibilityCheck): task=Tasks.text_error_correction, model=self.law_model_id) print(pipeline_ins(self.input_law)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_text_generation.py b/tests/pipelines/test_text_generation.py index 998cbd18..378b1bbc 100644 --- a/tests/pipelines/test_text_generation.py +++ b/tests/pipelines/test_text_generation.py @@ -8,11 +8,10 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import TextGenerationPipeline from modelscope.preprocessors import TextGenerationTransformersPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class TextGenerationTest(unittest.TestCase, DemoCompatibilityCheck): +class TextGenerationTest(unittest.TestCase): def setUp(self) -> None: self.palm_model_id_zh_base = 'damo/nlp_palm2.0_text-generation_chinese-base' @@ -261,10 +260,6 @@ class TextGenerationTest(unittest.TestCase, DemoCompatibilityCheck): model='damo/nlp_gpt2_text-generation_english-base') print(pipe('My name is Teven and I am')) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_text_to_image_synthesis.py b/tests/pipelines/test_text_to_image_synthesis.py index 5e28282b..63c38571 100644 --- a/tests/pipelines/test_text_to_image_synthesis.py +++ b/tests/pipelines/test_text_to_image_synthesis.py @@ -8,11 +8,10 @@ from modelscope.models import Model from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class TextToImageSynthesisTest(unittest.TestCase, DemoCompatibilityCheck): +class TextToImageSynthesisTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.text_to_image_synthesis @@ -61,10 +60,6 @@ class TextToImageSynthesisTest(unittest.TestCase, DemoCompatibilityCheck): self.test_text)[OutputKeys.OUTPUT_IMGS][0] print(np.sum(np.abs(img))) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_text_to_speech.py b/tests/pipelines/test_text_to_speech.py index f746dfbe..528977ce 100644 --- a/tests/pipelines/test_text_to_speech.py +++ b/tests/pipelines/test_text_to_speech.py @@ -11,7 +11,6 @@ import torch from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import test_level @@ -20,8 +19,7 @@ import tensorflow as tf # isort:skip logger = get_logger() -class TextToSpeechSambertHifigan16kPipelineTest(unittest.TestCase, - DemoCompatibilityCheck): +class TextToSpeechSambertHifigan16kPipelineTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.text_to_speech @@ -109,10 +107,6 @@ class TextToSpeechSambertHifigan16kPipelineTest(unittest.TestCase, with open(f'output_{self.test_model_name[i]}', 'wb') as f: f.write(wav) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_text_to_video_synthesis.py b/tests/pipelines/test_text_to_video_synthesis.py index 6463c155..97ef6089 100644 --- a/tests/pipelines/test_text_to_video_synthesis.py +++ b/tests/pipelines/test_text_to_video_synthesis.py @@ -5,11 +5,10 @@ import unittest from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class TextToVideoSynthesisTest(unittest.TestCase, DemoCompatibilityCheck): +class TextToVideoSynthesisTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.text_to_video_synthesis @@ -27,10 +26,6 @@ class TextToVideoSynthesisTest(unittest.TestCase, DemoCompatibilityCheck): self.test_text)[OutputKeys.OUTPUT_VIDEO] print(output_video_path) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_tinymog_face_detection.py b/tests/pipelines/test_tinymog_face_detection.py index e80fa482..48e74f44 100644 --- a/tests/pipelines/test_tinymog_face_detection.py +++ b/tests/pipelines/test_tinymog_face_detection.py @@ -8,11 +8,10 @@ from modelscope.msdatasets import MsDataset from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import draw_face_detection_result -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class TinyMogFaceDetectionTest(unittest.TestCase, DemoCompatibilityCheck): +class TinyMogFaceDetectionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.face_detection @@ -48,10 +47,6 @@ class TinyMogFaceDetectionTest(unittest.TestCase, DemoCompatibilityCheck): result = face_detection(self.img_path) self.show_result(self.img_path, result) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_tinynas_classification.py b/tests/pipelines/test_tinynas_classification.py index ebc6b722..300bd2b1 100644 --- a/tests/pipelines/test_tinynas_classification.py +++ b/tests/pipelines/test_tinynas_classification.py @@ -4,11 +4,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class TinyNASClassificationTest(unittest.TestCase, DemoCompatibilityCheck): +class TinyNASClassificationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_classification @@ -21,10 +20,6 @@ class TinyNASClassificationTest(unittest.TestCase, DemoCompatibilityCheck): result = tinynas_classification('data/test/images/image_wolf.jpeg') print(result) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_tinynas_detection.py b/tests/pipelines/test_tinynas_detection.py index f7c513ff..08c746ea 100644 --- a/tests/pipelines/test_tinynas_detection.py +++ b/tests/pipelines/test_tinynas_detection.py @@ -7,11 +7,10 @@ from PIL import Image from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class TinynasObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): +class TinynasObjectDetectionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_object_detection @@ -52,10 +51,6 @@ class TinynasObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): 'data/test/images/image_detection.jpg') print('damoyolo-t', result) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_image_object_detection_auto_pipeline(self): test_image = 'data/test/images/image_detection.jpg' diff --git a/tests/pipelines/test_traffic_sign_detection.py b/tests/pipelines/test_traffic_sign_detection.py index 5404649d..efedec14 100644 --- a/tests/pipelines/test_traffic_sign_detection.py +++ b/tests/pipelines/test_traffic_sign_detection.py @@ -7,20 +7,15 @@ from PIL import Image from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class TrafficSignDetectionTest(unittest.TestCase, DemoCompatibilityCheck): +class TrafficSignDetectionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.domain_specific_object_detection self.model_id = 'damo/cv_tinynas_object-detection_damoyolo_traffic_sign' - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_traffic_sign_detection_damoyolo(self): tinynas_object_detection = pipeline( diff --git a/tests/pipelines/test_translation_evaluation.py b/tests/pipelines/test_translation_evaluation.py index 8f16ea31..e936f41a 100644 --- a/tests/pipelines/test_translation_evaluation.py +++ b/tests/pipelines/test_translation_evaluation.py @@ -5,11 +5,10 @@ import unittest from modelscope.models.nlp.unite.configuration import InputFormat from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class TranslationEvaluationTest(unittest.TestCase, DemoCompatibilityCheck): +class TranslationEvaluationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.translation_evaluation diff --git a/tests/pipelines/test_translation_quality_estimation.py b/tests/pipelines/test_translation_quality_estimation.py index 315fa72b..0890f31b 100644 --- a/tests/pipelines/test_translation_quality_estimation.py +++ b/tests/pipelines/test_translation_quality_estimation.py @@ -3,12 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class TranslationQualityEstimationTest(unittest.TestCase, - DemoCompatibilityCheck): +class TranslationQualityEstimationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.sentence_similarity @@ -23,10 +21,6 @@ class TranslationQualityEstimationTest(unittest.TestCase, pipeline_ins = pipeline(self.task, model=self.model_id) print(pipeline_ins(input=inputs)) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_unifold.py b/tests/pipelines/test_unifold.py index cf67929d..98d2c1ce 100644 --- a/tests/pipelines/test_unifold.py +++ b/tests/pipelines/test_unifold.py @@ -4,11 +4,10 @@ import unittest from modelscope.hub.snapshot_download import snapshot_download from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class UnifoldProteinStructureTest(unittest.TestCase, DemoCompatibilityCheck): +class UnifoldProteinStructureTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.protein_structure diff --git a/tests/pipelines/test_universal_matting.py b/tests/pipelines/test_universal_matting.py index 5868cf36..1450d938 100644 --- a/tests/pipelines/test_universal_matting.py +++ b/tests/pipelines/test_universal_matting.py @@ -8,11 +8,10 @@ from modelscope.msdatasets import MsDataset from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import ModelFile, Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class UniversalMattingTest(unittest.TestCase, DemoCompatibilityCheck): +class UniversalMattingTest(unittest.TestCase): def setUp(self) -> None: self.model_id = 'damo/cv_unet_universal-matting' @@ -35,10 +34,6 @@ class UniversalMattingTest(unittest.TestCase, DemoCompatibilityCheck): cv2.imwrite('result.png', result[OutputKeys.OUTPUT_IMG]) print(f'Output written to {osp.abspath("result.png")}') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_user_satisfaction_estimation.py b/tests/pipelines/test_user_satisfaction_estimation.py index 2bbfd5d7..2904ea30 100644 --- a/tests/pipelines/test_user_satisfaction_estimation.py +++ b/tests/pipelines/test_user_satisfaction_estimation.py @@ -6,12 +6,10 @@ from modelscope.models import Model from modelscope.pipelines import pipeline from modelscope.preprocessors import DialogueClassificationUsePreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class UserSatisfactionEstimationTest(unittest.TestCase, - DemoCompatibilityCheck): +class UserSatisfactionEstimationTest(unittest.TestCase): model_id = 'damo/nlp_user-satisfaction-estimation_chinese' input_dialogue = [('返修退换货咨询|||', '手机有质量问题怎么办|||稍等，我看下', '开不开机了|||', @@ -33,10 +31,6 @@ class UserSatisfactionEstimationTest(unittest.TestCase, task=Tasks.text_classification, model=self.model_id) print(pipeline_ins(input=self.input_dialogue)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - print(self.compatibility_check()) - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_video_category.py b/tests/pipelines/test_video_category.py index 660196b8..61ee72b0 100644 --- a/tests/pipelines/test_video_category.py +++ b/tests/pipelines/test_video_category.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VideoCategoryTest(unittest.TestCase, DemoCompatibilityCheck): +class VideoCategoryTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_category @@ -21,10 +20,6 @@ class VideoCategoryTest(unittest.TestCase, DemoCompatibilityCheck): print(f'video category output: {result}.') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_video_colorization.py b/tests/pipelines/test_video_colorization.py index c35577a4..fe6c0f87 100644 --- a/tests/pipelines/test_video_colorization.py +++ b/tests/pipelines/test_video_colorization.py @@ -11,11 +11,10 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.base import Pipeline from modelscope.pipelines.cv import VideoColorizationPipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VideoColorizationTest(unittest.TestCase, DemoCompatibilityCheck): +class VideoColorizationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_colorization @@ -44,10 +43,6 @@ class VideoColorizationTest(unittest.TestCase, DemoCompatibilityCheck): video_colorization = pipeline(Tasks.video_colorization) self.pipeline_inference(video_colorization, self.test_video) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_video_deinterlace.py b/tests/pipelines/test_video_deinterlace.py index bcb36cc3..267d4664 100644 --- a/tests/pipelines/test_video_deinterlace.py +++ b/tests/pipelines/test_video_deinterlace.py @@ -8,11 +8,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.cv import VideoDeinterlacePipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VideoDeinterlaceTest(unittest.TestCase, DemoCompatibilityCheck): +class VideoDeinterlaceTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_deinterlace @@ -52,10 +51,6 @@ class VideoDeinterlaceTest(unittest.TestCase, DemoCompatibilityCheck): input=self.test_video)[OutputKeys.OUTPUT_VIDEO] print('pipeline: the output video path is {}'.format(out_video_path)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_video_depth_estimation.py b/tests/pipelines/test_video_depth_estimation.py index 30ca3b33..6a054b2b 100644 --- a/tests/pipelines/test_video_depth_estimation.py +++ b/tests/pipelines/test_video_depth_estimation.py @@ -5,11 +5,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import show_video_depth_estimation_result -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VideoDepthEstimationTest(unittest.TestCase, DemoCompatibilityCheck): +class VideoDepthEstimationTest(unittest.TestCase): def setUp(self) -> None: self.task = 'video-depth-estimation' diff --git a/tests/pipelines/test_video_frame_interpolation.py b/tests/pipelines/test_video_frame_interpolation.py index c23aa46a..11a4f568 100644 --- a/tests/pipelines/test_video_frame_interpolation.py +++ b/tests/pipelines/test_video_frame_interpolation.py @@ -8,11 +8,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.cv import VideoFrameInterpolationPipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VideoFrameInterpolationTest(unittest.TestCase, DemoCompatibilityCheck): +class VideoFrameInterpolationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_frame_interpolation @@ -58,10 +57,6 @@ class VideoFrameInterpolationTest(unittest.TestCase, DemoCompatibilityCheck): input=self.test_video)[OutputKeys.OUTPUT_VIDEO] print('pipeline: the output video path is {}'.format(out_video_path)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_video_instance_segmentation.py b/tests/pipelines/test_video_instance_segmentation.py index 0a76d260..465cf26f 100644 --- a/tests/pipelines/test_video_instance_segmentation.py +++ b/tests/pipelines/test_video_instance_segmentation.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VideoInstanceSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): +class VideoInstanceSegmentationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_panoptic_segmentation @@ -33,10 +32,6 @@ class VideoInstanceSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): print(f'video instance segmentation output:\n {result}.') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_video_multi_modal_embedding.py b/tests/pipelines/test_video_multi_modal_embedding.py index afe5940d..fe87b089 100644 --- a/tests/pipelines/test_video_multi_modal_embedding.py +++ b/tests/pipelines/test_video_multi_modal_embedding.py @@ -4,14 +4,13 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import test_level logger = get_logger() -class VideoMultiModalEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck): +class VideoMultiModalEmbeddingTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_multi_modal_embedding @@ -41,10 +40,6 @@ class VideoMultiModalEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck): logger.info('video feature: {}'.format( output['video_embedding'][0][0][0])) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_video_multi_object_tracking.py b/tests/pipelines/test_video_multi_object_tracking.py index 97f1e705..f63fd8b1 100644 --- a/tests/pipelines/test_video_multi_object_tracking.py +++ b/tests/pipelines/test_video_multi_object_tracking.py @@ -4,11 +4,10 @@ import unittest from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class MultiObjectTracking(unittest.TestCase, DemoCompatibilityCheck): +class MultiObjectTracking(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_multi_object_tracking @@ -34,10 +33,6 @@ class MultiObjectTracking(unittest.TestCase, DemoCompatibilityCheck): in result) assert len(result[OutputKeys.LABELS]) == len(result[OutputKeys.BOXES]) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_video_object_segmentation.py b/tests/pipelines/test_video_object_segmentation.py index e4adeb26..6f0e7c2a 100644 --- a/tests/pipelines/test_video_object_segmentation.py +++ b/tests/pipelines/test_video_object_segmentation.py @@ -9,11 +9,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import masks_visualization -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VideoObjectSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): +class VideoObjectSegmentationTest(unittest.TestCase): def setUp(self) -> None: self.task = 'video-object-segmentation' diff --git a/tests/pipelines/test_video_panoptic_segmentation.py b/tests/pipelines/test_video_panoptic_segmentation.py index ad038135..cc805812 100644 --- a/tests/pipelines/test_video_panoptic_segmentation.py +++ b/tests/pipelines/test_video_panoptic_segmentation.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VideoPanopticSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): +class VideoPanopticSegmentationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_panoptic_segmentation @@ -32,10 +31,6 @@ class VideoPanopticSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): print(f'video summarization output:\n {result}.') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_video_single_object_tracking.py b/tests/pipelines/test_video_single_object_tracking.py index e75ccbb0..c8331649 100644 --- a/tests/pipelines/test_video_single_object_tracking.py +++ b/tests/pipelines/test_video_single_object_tracking.py @@ -5,11 +5,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import show_video_tracking_result -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class SingleObjectTracking(unittest.TestCase, DemoCompatibilityCheck): +class SingleObjectTracking(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_single_object_tracking @@ -46,10 +45,6 @@ class SingleObjectTracking(unittest.TestCase, DemoCompatibilityCheck): result = video_single_object_tracking((video_path, init_bbox)) print('result is : ', result[OutputKeys.BOXES]) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_video_stabilization.py b/tests/pipelines/test_video_stabilization.py index d102f3e1..26501c2d 100644 --- a/tests/pipelines/test_video_stabilization.py +++ b/tests/pipelines/test_video_stabilization.py @@ -7,11 +7,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.cv import VideoStabilizationPipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VideoStabilizationTest(unittest.TestCase, DemoCompatibilityCheck): +class VideoStabilizationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_stabilization @@ -42,10 +41,6 @@ class VideoStabilizationTest(unittest.TestCase, DemoCompatibilityCheck): input=self.test_video)[OutputKeys.OUTPUT_VIDEO] print('pipeline: the output video path is {}'.format(out_video_path)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_video_summarization.py b/tests/pipelines/test_video_summarization.py index 1f965c53..dc6a3a80 100644 --- a/tests/pipelines/test_video_summarization.py +++ b/tests/pipelines/test_video_summarization.py @@ -3,11 +3,10 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VideoSummarizationTest(unittest.TestCase, DemoCompatibilityCheck): +class VideoSummarizationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_summarization @@ -30,10 +29,6 @@ class VideoSummarizationTest(unittest.TestCase, DemoCompatibilityCheck): print(f'video summarization output:\n {result}.') - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_video_super_resolution.py b/tests/pipelines/test_video_super_resolution.py index 0da18dd7..2e207887 100644 --- a/tests/pipelines/test_video_super_resolution.py +++ b/tests/pipelines/test_video_super_resolution.py @@ -7,11 +7,10 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.pipelines.cv import VideoSuperResolutionPipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VideoSuperResolutionTest(unittest.TestCase, DemoCompatibilityCheck): +class VideoSuperResolutionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.video_super_resolution @@ -50,10 +49,6 @@ class VideoSuperResolutionTest(unittest.TestCase, DemoCompatibilityCheck): input=self.test_video)[OutputKeys.OUTPUT_VIDEO] print('pipeline: the output video path is {}'.format(out_video_path)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_vidt_face.py b/tests/pipelines/test_vidt_face.py index 8640d128..e49d9de9 100644 --- a/tests/pipelines/test_vidt_face.py +++ b/tests/pipelines/test_vidt_face.py @@ -5,11 +5,10 @@ from modelscope.models import Model from modelscope.models.cv.vidt import VidtModel from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VidtTest(unittest.TestCase, DemoCompatibilityCheck): +class VidtTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_object_detection diff --git a/tests/pipelines/test_vidt_logo.py b/tests/pipelines/test_vidt_logo.py index 143eb205..fce6fe48 100644 --- a/tests/pipelines/test_vidt_logo.py +++ b/tests/pipelines/test_vidt_logo.py @@ -5,11 +5,10 @@ from modelscope.models import Model from modelscope.models.cv.vidt import VidtModel from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VidtTest(unittest.TestCase, DemoCompatibilityCheck): +class VidtTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_object_detection diff --git a/tests/pipelines/test_virtual_try_on.py b/tests/pipelines/test_virtual_try_on.py index 5c18dcc4..c8a55f79 100644 --- a/tests/pipelines/test_virtual_try_on.py +++ b/tests/pipelines/test_virtual_try_on.py @@ -8,11 +8,10 @@ from PIL import Image from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VirtualTryonTest(unittest.TestCase, DemoCompatibilityCheck): +class VirtualTryonTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.virtual_try_on @@ -36,10 +35,6 @@ class VirtualTryonTest(unittest.TestCase, DemoCompatibilityCheck): img = pipeline_virtual_tryon(self.input_imgs)[OutputKeys.OUTPUT_IMG] cv2.imwrite('demo.jpg', img[:, :, ::-1]) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_vision_efficient_tuning.py b/tests/pipelines/test_vision_efficient_tuning.py index c88ed478..acfbb235 100644 --- a/tests/pipelines/test_vision_efficient_tuning.py +++ b/tests/pipelines/test_vision_efficient_tuning.py @@ -6,11 +6,10 @@ from modelscope.models.cv.vision_efficient_tuning.model import \ VisionEfficientTuningModel from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VisionEfficientTuningTest(unittest.TestCase, DemoCompatibilityCheck): +class VisionEfficientTuningTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.vision_efficient_tuning @@ -29,11 +28,6 @@ class VisionEfficientTuningTest(unittest.TestCase, DemoCompatibilityCheck): model = Model.from_pretrained(model_id) self.assertTrue(model.__class__ == VisionEfficientTuningModel) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_vision_efficient_tuning_adapter_demo_compatibility(self): - self.model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-adapter' - self.compatibility_check() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_vision_efficient_tuning_lora_run_pipeline(self): model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-lora' @@ -48,11 +42,6 @@ class VisionEfficientTuningTest(unittest.TestCase, DemoCompatibilityCheck): model = Model.from_pretrained(model_id) self.assertTrue(model.__class__ == VisionEfficientTuningModel) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_vision_efficient_tuning_lora_demo_compatibility(self): - self.model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-lora' - self.compatibility_check() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_vision_efficient_tuning_prefix_run_pipeline(self): model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-prefix' @@ -67,11 +56,6 @@ class VisionEfficientTuningTest(unittest.TestCase, DemoCompatibilityCheck): model = Model.from_pretrained(model_id) self.assertTrue(model.__class__ == VisionEfficientTuningModel) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_vision_efficient_tuning_prefix_demo_compatibility(self): - self.model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-prefix' - self.compatibility_check() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_vision_efficient_tuning_prompt_run_pipeline(self): model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-prompt' @@ -86,11 +70,6 @@ class VisionEfficientTuningTest(unittest.TestCase, DemoCompatibilityCheck): model = Model.from_pretrained(model_id) self.assertTrue(model.__class__ == VisionEfficientTuningModel) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_vision_efficient_tuning_prompt_demo_compatibility(self): - self.model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-prompt' - self.compatibility_check() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_vision_efficient_tuning_bitfit_run_pipeline(self): model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-bitfit' @@ -105,11 +84,6 @@ class VisionEfficientTuningTest(unittest.TestCase, DemoCompatibilityCheck): model = Model.from_pretrained(model_id) self.assertTrue(model.__class__ == VisionEfficientTuningModel) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_vision_efficient_tuning_bitfit_demo_compatibility(self): - self.model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-bitfit' - self.compatibility_check() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_vision_efficient_tuning_sidetuning_run_pipeline(self): model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-sidetuning' @@ -125,11 +99,6 @@ class VisionEfficientTuningTest(unittest.TestCase, DemoCompatibilityCheck): model = Model.from_pretrained(model_id) self.assertTrue(model.__class__ == VisionEfficientTuningModel) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_vision_efficient_tuning_sidetuning_demo_compatibility(self): - self.model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-sidetuning' - self.compatibility_check() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_vision_efficient_tuning_utuning_run_pipeline(self): model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-utuning' @@ -144,11 +113,6 @@ class VisionEfficientTuningTest(unittest.TestCase, DemoCompatibilityCheck): model = Model.from_pretrained(model_id) self.assertTrue(model.__class__ == VisionEfficientTuningModel) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_vision_efficient_tuning_utuning_demo_compatibility(self): - self.model_id = 'damo/cv_vitb16_classification_vision-efficient-tuning-utuning' - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_vision_middleware.py b/tests/pipelines/test_vision_middleware.py index b3531154..e8c1218b 100644 --- a/tests/pipelines/test_vision_middleware.py +++ b/tests/pipelines/test_vision_middleware.py @@ -5,11 +5,10 @@ from modelscope.models import Model from modelscope.models.cv.vision_middleware import VisionMiddlewareModel from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VisionMiddlewareTest(unittest.TestCase, DemoCompatibilityCheck): +class VisionMiddlewareTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_segmentation diff --git a/tests/pipelines/test_vop_retrieval.py b/tests/pipelines/test_vop_retrieval.py index c9c356c5..78e7eecc 100644 --- a/tests/pipelines/test_vop_retrieval.py +++ b/tests/pipelines/test_vop_retrieval.py @@ -5,11 +5,10 @@ from modelscope.models import Model from modelscope.models.cv.vop_retrieval import VoP from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VopRetrievalTest(unittest.TestCase, DemoCompatibilityCheck): +class VopRetrievalTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.vop_retrieval diff --git a/tests/pipelines/test_vop_retrieval_sebias.py b/tests/pipelines/test_vop_retrieval_sebias.py index bea1bc45..a129f7f0 100644 --- a/tests/pipelines/test_vop_retrieval_sebias.py +++ b/tests/pipelines/test_vop_retrieval_sebias.py @@ -5,11 +5,10 @@ from modelscope.models import Model from modelscope.models.cv.vop_retrieval import VideoTextRetrievalModelSeries from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VopRetrievalTest(unittest.TestCase, DemoCompatibilityCheck): +class VopRetrievalTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.vop_retrieval diff --git a/tests/pipelines/test_vop_retrieval_separtial.py b/tests/pipelines/test_vop_retrieval_separtial.py index 942fbd3b..c5832aaa 100644 --- a/tests/pipelines/test_vop_retrieval_separtial.py +++ b/tests/pipelines/test_vop_retrieval_separtial.py @@ -5,11 +5,10 @@ from modelscope.models import Model from modelscope.models.cv.vop_retrieval import VideoTextRetrievalModelSeries from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VopRetrievalTest(unittest.TestCase, DemoCompatibilityCheck): +class VopRetrievalTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.vop_retrieval diff --git a/tests/pipelines/test_vop_retrieval_seproj.py b/tests/pipelines/test_vop_retrieval_seproj.py index a371ac36..2fceb2e7 100644 --- a/tests/pipelines/test_vop_retrieval_seproj.py +++ b/tests/pipelines/test_vop_retrieval_seproj.py @@ -5,11 +5,10 @@ from modelscope.models import Model from modelscope.models.cv.vop_retrieval import VideoTextRetrievalModelSeries from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class VopRetrievalTest(unittest.TestCase, DemoCompatibilityCheck): +class VopRetrievalTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.vop_retrieval diff --git a/tests/pipelines/test_wenet_automatic_speech_recognition.py b/tests/pipelines/test_wenet_automatic_speech_recognition.py index 4adf8119..ac47cea7 100644 --- a/tests/pipelines/test_wenet_automatic_speech_recognition.py +++ b/tests/pipelines/test_wenet_automatic_speech_recognition.py @@ -10,7 +10,6 @@ import soundfile from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import ColorCodes, Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import download_and_untar, test_level @@ -20,8 +19,7 @@ WAV_FILE = 'data/test/audios/asr_example.wav' URL_FILE = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example.wav' -class WeNetAutomaticSpeechRecognitionTest(unittest.TestCase, - DemoCompatibilityCheck): +class WeNetAutomaticSpeechRecognitionTest(unittest.TestCase): action_info = { 'test_run_with_pcm': { 'checking_item': OutputKeys.TEXT, diff --git a/tests/pipelines/test_word_segmentation.py b/tests/pipelines/test_word_segmentation.py index f8bdaef7..f8c9e078 100644 --- a/tests/pipelines/test_word_segmentation.py +++ b/tests/pipelines/test_word_segmentation.py @@ -10,12 +10,11 @@ from modelscope.pipelines.nlp import WordSegmentationPipeline from modelscope.preprocessors import \ TokenClassificationTransformersPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.regress_test_utils import IgnoreKeyFn, MsRegressTool from modelscope.utils.test_utils import test_level -class WordSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): +class WordSegmentationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.word_segmentation @@ -164,10 +163,6 @@ class WordSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): pipeline_ins = pipeline(task=Tasks.word_segmentation) print(pipeline_ins(input=self.sentence)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_zero_shot_classification.py b/tests/pipelines/test_zero_shot_classification.py index f9a52b42..89832d18 100644 --- a/tests/pipelines/test_zero_shot_classification.py +++ b/tests/pipelines/test_zero_shot_classification.py @@ -9,12 +9,11 @@ from modelscope.pipelines.nlp import ZeroShotClassificationPipeline from modelscope.preprocessors import \ ZeroShotClassificationTransformersPreprocessor from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.regress_test_utils import IgnoreKeyFn, MsRegressTool from modelscope.utils.test_utils import test_level -class ZeroShotClassificationTest(unittest.TestCase, DemoCompatibilityCheck): +class ZeroShotClassificationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.zero_shot_classification @@ -79,10 +78,6 @@ class ZeroShotClassificationTest(unittest.TestCase, DemoCompatibilityCheck): pipeline_ins = pipeline(task=Tasks.zero_shot_classification) print(pipeline_ins(input=self.sentence, candidate_labels=self.labels)) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() From 1d28c5b73082329ed60f463ddf08f05a62e27878 Mon Sep 17 00:00:00 2001 From: "hemu.zp" Date: Mon, 15 May 2023 11:48:29 +0800 Subject: [PATCH 37/76] mv mmdet to cv Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12614719 --- requirements/cv.txt | 2 +- requirements/framework.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements/cv.txt b/requirements/cv.txt index 3230d3fe..6ea02c54 100644 --- a/requirements/cv.txt +++ b/requirements/cv.txt @@ -25,7 +25,7 @@ lmdb lpips ml_collections mmcls>=0.21.0 -mmdet>=2.25.0 +mmdet>=2.25.0,<=2.28.2 # mmdet3d-1.0.0rc6 remove networkx and numba version restriction mmdet3d==1.0.0a1 mmsegmentation<=0.30.0 diff --git a/requirements/framework.txt b/requirements/framework.txt index 88d60bc1..03edde4d 100644 --- a/requirements/framework.txt +++ b/requirements/framework.txt @@ -4,7 +4,6 @@ datasets>=2.7.0,<=2.8.0 einops filelock>=3.3.0 gast>=0.2.2 -mmdet<=2.28.2 numpy<1.24.0 oss2 # for datasets compatible From b9c8c99776f97d65f941f6e694d66b05058a011e Mon Sep 17 00:00:00 2001 From: "yeqinghao.yqh" Date: Mon, 15 May 2023 16:32:46 +0800 Subject: [PATCH 38/76] Support mPLUG-Owl model. Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12610417 --- data/test | 2 +- modelscope/metainfo.py | 3 + modelscope/models/multi_modal/__init__.py | 2 + .../models/multi_modal/mplug_owl/__init__.py | 18 + .../mplug_owl/configuration_mplug_owl.py | 257 +++ .../mplug_owl/modeling_mplug_owl.py | 1551 +++++++++++++++++ modelscope/outputs/outputs.py | 4 + modelscope/pipeline_inputs.py | 3 + modelscope/pipelines/multi_modal/__init__.py | 2 + .../multimodal_dialogue_pipeline.py | 90 + modelscope/preprocessors/__init__.py | 4 +- modelscope/preprocessors/multi_modal.py | 145 +- modelscope/utils/constant.py | 1 + tests/pipelines/test_clip_interrogator.py | 9 +- .../test_face_recognition_onnx_transface.py | 7 +- .../test_fast_instance_segmentation.py | 7 +- .../test_mplug_owl_multimodal_dialogue.py | 100 ++ tests/pipelines/test_speaker_verification.py | 5 - 18 files changed, 2181 insertions(+), 29 deletions(-) create mode 100644 modelscope/models/multi_modal/mplug_owl/__init__.py create mode 100644 modelscope/models/multi_modal/mplug_owl/configuration_mplug_owl.py create mode 100644 modelscope/models/multi_modal/mplug_owl/modeling_mplug_owl.py create mode 100644 modelscope/pipelines/multi_modal/multimodal_dialogue_pipeline.py create mode 100644 tests/pipelines/test_mplug_owl_multimodal_dialogue.py diff --git a/data/test b/data/test index 91b37f8d..8d062525 160000 --- a/data/test +++ b/data/test @@ -1 +1 @@ -Subproject commit 91b37f8d6251089aa878520f389fbefc2ce9ffc4 +Subproject commit 8d0625256b88bdf41655563049a4a68ec1025638 diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index efce05b0..2e3fa39d 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -202,6 +202,7 @@ class Models(object): hitea = 'hitea' soonet = 'soonet' efficient_diffusion_tuning = 'efficient-diffusion-tuning' + mplug_owl = 'mplug-owl' clip_interrogator = 'clip-interrogator' # science models @@ -512,6 +513,7 @@ class Pipelines(object): gridvlp_multi_modal_embedding = 'gridvlp-multi-modal-embedding' soonet_video_temporal_grounding = 'soonet-video-temporal-grounding' efficient_diffusion_tuning = 'efficient-diffusion-tuning' + multimodal_dialogue = 'multimodal-dialogue' # science tasks protein_structure = 'unifold-protein-structure' @@ -1030,6 +1032,7 @@ class Preprocessors(object): vldoc_preprocessor = 'vldoc-preprocessor' hitea_tasks_preprocessor = 'hitea-tasks-preprocessor' diffusion_image_generation_preprocessor = 'diffusion-image-generation-preprocessor' + mplug_owl_preprocessor = 'mplug-owl-preprocessor' image_captioning_clip_interrogator_preprocessor = 'image-captioning-clip-interrogator-preprocessor' # science preprocessor diff --git a/modelscope/models/multi_modal/__init__.py b/modelscope/models/multi_modal/__init__.py index cfcc0361..9fa34baf 100644 --- a/modelscope/models/multi_modal/__init__.py +++ b/modelscope/models/multi_modal/__init__.py @@ -20,6 +20,7 @@ if TYPE_CHECKING: from .vldoc import VLDocForDocVLEmbedding from .video_synthesis import TextToVideoSynthesis from .efficient_diffusion_tuning import EfficientStableDiffusion + from .mplug_owl import MplugOwlForConditionalGeneration from .clip_interrogator import CLIP_Interrogator else: @@ -39,6 +40,7 @@ else: 'vldoc': ['VLDocForDocVLEmbedding'], 'video_synthesis': ['TextToVideoSynthesis'], 'efficient_diffusion_tuning': ['EfficientStableDiffusion'], + 'mplug_owl': ['MplugOwlForConditionalGeneration'], 'clip_interrogator': ['CLIP_Interrogator'], } diff --git a/modelscope/models/multi_modal/mplug_owl/__init__.py b/modelscope/models/multi_modal/mplug_owl/__init__.py new file mode 100644 index 00000000..76ccfb5a --- /dev/null +++ b/modelscope/models/multi_modal/mplug_owl/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2021-2023 The Alibaba DAMO mPLUG Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .configuration_mplug_owl import (MplugOwlConfig, MplugOwlVisionConfig, + MplugOwlVisualAbstractorConfig) +from .modeling_mplug_owl import MplugOwlForConditionalGeneration diff --git a/modelscope/models/multi_modal/mplug_owl/configuration_mplug_owl.py b/modelscope/models/multi_modal/mplug_owl/configuration_mplug_owl.py new file mode 100644 index 00000000..6e32238a --- /dev/null +++ b/modelscope/models/multi_modal/mplug_owl/configuration_mplug_owl.py @@ -0,0 +1,257 @@ +# Copyright 2021-2023 The Alibaba DAMO mPLUG Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" MPLUG OWL model configuration """ +import copy +import os +from typing import Union + +from transformers import PretrainedConfig +from transformers.models.auto import CONFIG_MAPPING +from transformers.utils import logging + +from modelscope.utils.constant import Tasks + +logger = logging.get_logger() + + +class MplugOwlVisionConfig(PretrainedConfig): + r""" + Args: + hidden_size (`int`, *optional*, defaults to 768): + Dimensionality of the encoder layers and the pooler layer. + intermediate_size (`int`, *optional*, defaults to 3072): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + num_hidden_layers (`int`, *optional*, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 12): + Number of attention heads for each attention layer in the Transformer encoder. + image_size (`int`, *optional*, defaults to 224): + The size (resolution) of each image. + patch_size (`int`, *optional*, defaults to 32): + The size (resolution) of each patch. + hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported. + layer_norm_eps (`float`, *optional*, defaults to 1e-5): + The epsilon used by the layer normalization layers. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + initializer_factor (`float`, *optional*, defaults to 1): + A factor for initializing all weight matrices (should be kept to 1, used internally for initialization + testing). + ```""" + + model_type = 'mplug_owl_vision_model' + + def __init__( + self, + hidden_size=1024, + intermediate_size=4096, + projection_dim=768, + num_hidden_layers=24, + num_attention_heads=16, + num_channels=3, + image_size=224, + patch_size=14, + hidden_act='quick_gelu', + layer_norm_eps=1e-6, + attention_dropout=0.0, + initializer_range=0.02, + initializer_factor=1.0, + use_flash_attn=False, + **kwargs, + ): + super().__init__(**kwargs) + + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.projection_dim = projection_dim + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_channels = num_channels + self.patch_size = patch_size + self.image_size = image_size + self.initializer_range = initializer_range + self.initializer_factor = initializer_factor + self.attention_dropout = attention_dropout + self.layer_norm_eps = layer_norm_eps + self.hidden_act = hidden_act + self.use_flash_attn = use_flash_attn + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: Union[str, + os.PathLike], + **kwargs) -> 'PretrainedConfig': + config_dict, kwargs = cls.get_config_dict( + pretrained_model_name_or_path, **kwargs) + + # get the vision config dict if we are loading from MplugOwlConfig + if config_dict.get('model_type') == 'mplug_owl': + config_dict = config_dict['vision_config'] + + if 'model_type' in config_dict and hasattr( + cls, + 'model_type') and config_dict['model_type'] != cls.model_type: + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.' + ) + + return cls.from_dict(config_dict, **kwargs) + + +class MplugOwlVisualAbstractorConfig(PretrainedConfig): + + model_type = 'MPlugOwlVisualAbstractor' + + def __init__( + self, + hidden_size=1024, + num_hidden_layers=6, + num_attention_heads=16, + intermediate_size=4096, + attention_probs_dropout_prob=0.1, + initializer_range=0.02, + layer_norm_eps=1e-6, + encoder_hidden_size=1024, + **kwargs, + ): + super().__init__(**kwargs) + + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + self.encoder_hidden_size = encoder_hidden_size + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: Union[str, + os.PathLike], + **kwargs) -> 'PretrainedConfig': + config_dict, kwargs = cls.get_config_dict( + pretrained_model_name_or_path, **kwargs) + + # get the qformer config dict if we are loading from MplugOwlConfig + if config_dict.get('model_type') == 'mplug_owl': + config_dict = config_dict['abstractor_config'] + + if 'model_type' in config_dict and hasattr( + cls, + 'model_type') and config_dict['model_type'] != cls.model_type: + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.' + ) + + return cls.from_dict(config_dict, **kwargs) + + +class MplugOwlConfig(PretrainedConfig): + r""" + Args: + vision_config (`dict`, *optional*): + Dictionary of configuration options used to initialize [`MplugOwlVisionConfig`]. + qformer_config (`dict`, *optional*): + Dictionary of configuration options used to initialize [`MplugOwlVisualAbstractorConfig`]. + text_config (`dict`, *optional*): + Dictionary of configuration options used to initialize any [`PretrainedConfig`]. + num_query_tokens (`int`, *optional*, defaults to 32): + The number of query tokens passed through the Transformer. + + kwargs (*optional*): + Dictionary of keyword arguments. + """ + + model_type = 'mplug_owl' + is_composition = True + + def __init__(self, + task=Tasks.multimodal_dialogue, + vision_config=None, + visual_abstractor_config=None, + text_config=None, + num_query_tokens=64, + **kwargs): + + super().__init__(**kwargs) + self.task = task + if vision_config is None: + vision_config = MplugOwlVisionConfig().to_dict() + logger.info('vision_config is None.') + + if visual_abstractor_config is None: + visual_abstractor_config = {} + logger.info('abstractor_config is None. ') + + if text_config is None: + # we use LLAMA 7b by default + from transformers.models.llama.configuration_llama import \ + LlamaConfig + text_config = LlamaConfig(pad_token_id=2).to_dict() + logger.info('text_config is None.') + + self.vision_config = MplugOwlVisionConfig(**vision_config) + self.visual_abstractor_config = MplugOwlVisualAbstractorConfig( + **visual_abstractor_config) + text_model_type = text_config[ + 'model_type'] if 'model_type' in text_config else 'llama' + self.text_config = CONFIG_MAPPING[text_model_type](**text_config) + + self.tie_word_embeddings = self.text_config.tie_word_embeddings + + self.num_query_tokens = num_query_tokens + self.initializer_factor = 1.0 + self.initializer_range = 0.02 + + @classmethod + def from_vision_abstractor_text_configs( + cls, + vision_config: MplugOwlVisionConfig, + visual_abstractor_config: MplugOwlVisualAbstractorConfig, + text_config: PretrainedConfig, + **kwargs, + ): + r""" + Returns: + [`MplugOwlConfig`]: An instance of a configuration object + """ + + return cls( + vision_config=vision_config.to_dict(), + visual_abstractor_config=visual_abstractor_config.to_dict(), + text_config=text_config.to_dict(), + **kwargs, + ) + + def to_dict(self): + """ + Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`]. + + Returns: + `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, + """ + output = copy.deepcopy(self.__dict__) + output['vision_config'] = self.vision_config.to_dict() + tmp = self.visual_abstractor_config.to_dict() + output['visual_abstractor_config'] = tmp + output['text_config'] = self.text_config.to_dict() + output['model_type'] = self.__class__.model_type + return output diff --git a/modelscope/models/multi_modal/mplug_owl/modeling_mplug_owl.py b/modelscope/models/multi_modal/mplug_owl/modeling_mplug_owl.py new file mode 100644 index 00000000..21a29185 --- /dev/null +++ b/modelscope/models/multi_modal/mplug_owl/modeling_mplug_owl.py @@ -0,0 +1,1551 @@ +# Copyright 2021-2023 The Alibaba DAMO mPLUG Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch MPLUG OWL model. """ + +import copy +import logging +import math +import os +import os.path as osp +import random +from dataclasses import dataclass +from io import BytesIO +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint +import transformers +from torch.nn import CrossEntropyLoss +from transformers.activations import ACT2FN +from transformers.modeling_outputs import ( + BaseModelOutput, BaseModelOutputWithPastAndCrossAttentions, + BaseModelOutputWithPooling, BaseModelOutputWithPoolingAndCrossAttentions, + CausalLMOutputWithCrossAttentions) +from transformers.modeling_utils import (PreTrainedModel, + apply_chunking_to_forward, + find_pruneable_heads_and_indices, + prune_linear_layer) +from transformers.models.auto import AutoModelForCausalLM +from transformers.utils import ModelOutput + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.base import Tensor +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.mplug_owl.configuration_mplug_owl import ( + MplugOwlConfig, MplugOwlVisionConfig, MplugOwlVisualAbstractorConfig) +from modelscope.outputs import OutputKeys +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks + +__all__ = ['MplugOwlForConditionalGeneration'] + + +@dataclass +class MplugOwlForConditionalGenerationModelOutput(ModelOutput): + """ + Class defining the outputs of [`MPlugOwlForConditionalGeneration`]. + + Args: + loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): + Language modeling loss from the language model. + logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): + Prediction scores of the language modeling head of the language model. + vision_outputs (`BaseModelOutputWithPooling`): + Outputs of the vision encoder. + + language_model_outputs (`CausalLMOutputWithPast`): + Outputs of the language model. + """ + + loss: Optional[Tuple[torch.FloatTensor]] = None + logits: Optional[Tuple[torch.FloatTensor]] = None + vision_outputs: Optional[torch.FloatTensor] = None + language_model_outputs: Optional[Tuple[torch.FloatTensor]] = None + + def to_tuple(self) -> Tuple[Any]: + return tuple( + self[k] if k not in ['vision_outputs', 'language_model_outputs' + ] else getattr(self, k).to_tuple() + for k in self.keys()) + + +def get_ltor_masks_and_position_ids_from_embeddings(data): + """Build masks and position id for left to right model.""" + + # Extract batch size and sequence length. + micro_batch_size, seq_length = data.size()[:2] + + # Attention mask (lower triangular). + att_mask_batch = 1 + attention_mask = torch.tril( + torch.ones((att_mask_batch, seq_length, seq_length), + device=data.device)).view(att_mask_batch, 1, seq_length, + seq_length) + + # Loss mask. + loss_mask = torch.ones( + data.size()[:2], dtype=torch.float, device=data.device) + + # Position ids. + position_ids = torch.arange( + seq_length, dtype=torch.long, device=data.device) + position_ids = position_ids.unsqueeze(0).expand_as(data[..., 0]) + + # Convert attention mask to binary: + attention_mask = (attention_mask < 0.5) + + return attention_mask, loss_mask, position_ids + + +class MplugOwlVisionEmbeddings(nn.Module): + + def __init__(self, config: MplugOwlVisionConfig): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.cls_token = nn.Parameter(torch.randn(1, 1, self.hidden_size)) + + self.patch_embed = nn.Conv2d( + in_channels=3, + out_channels=self.hidden_size, + kernel_size=self.patch_size, + stride=self.patch_size, + bias=False) + + self.num_patches = (self.image_size // self.patch_size)**2 + + self.position_embedding = nn.Parameter( + torch.randn(1, self.num_patches + 1, self.hidden_size)) + + self.pre_layernorm = LayerNormFp32( + self.hidden_size, eps=config.layer_norm_eps) + + def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: + batch_size = pixel_values.size(0) + image_embeds = self.patch_embed(pixel_values) + image_embeds = image_embeds.flatten(2).transpose(1, 2) + + class_embeds = self.cls_token.expand(batch_size, 1, + -1).to(image_embeds.dtype) + embeddings = torch.cat([class_embeds, image_embeds], dim=1) + embeddings = embeddings + \ + self.position_embedding[:, : embeddings.size(1)].to( + image_embeds.dtype) + embeddings = self.pre_layernorm(embeddings) + return embeddings + + +class LayerNormFp32(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16 (by casting to float32 and back).""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def forward(self, x: torch.Tensor): + output = torch.nn.functional.layer_norm( + x.float(), + self.normalized_shape, + self.weight.float() if self.weight is not None else None, + self.bias.float() if self.bias is not None else None, + self.eps, + ) + return output.type_as(x) + + +class MplugOwlVisionAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + if self.head_dim * self.num_heads != self.hidden_size: + raise ValueError( + f'hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size} and `num_heads`:' + f' {self.num_heads}).') + self.scale = self.head_dim**-0.5 + self.dropout = nn.Dropout(config.attention_dropout) + + self.query_key_value = nn.Linear(self.hidden_size, + 3 * self.hidden_size) + self.dense = nn.Linear(self.hidden_size, self.hidden_size) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, + self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + head_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], + Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + bsz, seq_len, embed_dim = hidden_states.size() + + mixed_qkv = self.query_key_value(hidden_states) + + mixed_qkv = mixed_qkv.reshape(bsz, seq_len, self.num_heads, 3, + embed_dim // self.num_heads).permute( + 3, 0, 2, 1, 4) # [3, b, np, sq, hn] + query_states, key_states, value_states = ( + mixed_qkv[0], + mixed_qkv[1], + mixed_qkv[2], + ) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_states, + key_states.transpose(-1, -2)) + + attention_scores = attention_scores * self.scale + + # Normalize the attention scores to probabilities. + attention_probs = torch.softmax(attention_scores, dim=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, + value_states).permute(0, 2, 1, 3) + + new_context_layer_shape = context_layer.size()[:-2] + ( + self.hidden_size, ) + context_layer = context_layer.reshape(new_context_layer_shape) + + output = self.dense(context_layer) + + outputs = (output, attention_probs) if output_attentions else (output, + None) + + return outputs + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class MplugOwlMLP(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.activation_fn = QuickGELU() + self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size) + self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states = self.fc2(hidden_states) + return hidden_states + + +class MplugOwlVisionEncoderLayer(nn.Module): + + def __init__(self, config: MplugOwlVisionConfig): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = MplugOwlVisionAttention(config) + self.input_layernorm = LayerNormFp32( + self.hidden_size, eps=config.layer_norm_eps) + self.mlp = MplugOwlMLP(config) + self.post_attention_layernorm = LayerNormFp32( + self.hidden_size, eps=config.layer_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.FloatTensor]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + `(config.encoder_attention_heads,)`. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + """ + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + hidden_states, attn_weights = self.self_attn( + hidden_states=hidden_states, + head_mask=attention_mask, + output_attentions=output_attentions, + ) + hidden_states = hidden_states + residual + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + + hidden_states = hidden_states + residual + + outputs = (hidden_states, ) + + if output_attentions: + outputs += (attn_weights, ) + + return outputs + + +class MplugOwlPreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = MplugOwlConfig + base_model_prefix = 'mplug_owl' + supports_gradient_checkpointing = True + _keys_to_ignore_on_load_missing = [ + r'position_ids', + r'language_model.encoder.embed_tokens.weight', + r'language_model.decoder.embed_tokens.weight', + r'language_model.lm_head.weight', + ] + _no_split_modules = ['MplugOwlAttention'] + _keep_in_fp32_modules = ['wo'] + + def _init_weights(self, module): + """Initialize the weights""" + factor = self.config.initializer_range + if isinstance(module, nn.Conv2d) or isinstance( + module, nn.Embedding) or isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=factor) + if hasattr(module, 'bias') and module.bias is not None: + module.bias.data.zero_() + + if isinstance(module, MplugOwlVisionEmbeddings): + if hasattr(self.config, 'vision_config'): + factor = self.config.vision_config.initializer_range + nn.init.trunc_normal_( + module.position_embedding, mean=0.0, std=factor) + nn.init.trunc_normal_(module.cls_token, mean=0.0, std=factor) + + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + elif isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Parameter): + nn.init.trunc_normal_(module.data, mean=0.0, std=factor) + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, MplugOwlVisionEncoder): + module.gradient_checkpointing = value + + +MPLUG_OWL_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`MplugOwlConfig`]): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + +MPLUG_OWL_VISION_INPUTS_DOCSTRING = r""" + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Pixel values. Pixel values can be obtained using [`MplugOwlPreprocessor`]. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + +MPLUG_OWL_TEXT_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + [What are attention masks?](../glossary#attention-mask) + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 + Training](./t5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + +MPLUG_OWL_INPUTS_DOCSTRING = r""" + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Pixel values. Pixel values can be obtained using [`MplugOwlPreprocessor`]. + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be + provided to serve as text prompt, which the language model can continue. + + Indices can be obtained using [`MplugOwlPreprocessor`]. See [`MplugOwlPreprocessor.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary of the language model. Only relevant in case an + encoder-decoder language model (like T5) is used. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) + + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + Only relevant in case an encoder-decoder language model (like T5) is used. + + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +class MplugOwlVisionEncoder(nn.Module): + """ + Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a + [`MplugOwlVisionEncoderLayer`]. + + Args: + config (`MplugOwlVisionConfig`): + The corresponding vision configuration for the `MplugOwlEncoder`. + """ + + def __init__(self, config: MplugOwlVisionConfig): + super().__init__() + self.config = config + self.layers = nn.ModuleList([ + MplugOwlVisionEncoderLayer(config) + for _ in range(config.num_hidden_layers) + ]) + self.gradient_checkpointing = False + + def forward( + self, + inputs_embeds, + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutput]: + r""" + Args: + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Embedded representation of the inputs. Should be float, not int tokens. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + encoder_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + + hidden_states = inputs_embeds + for idx, encoder_layer in enumerate(self.layers): + if output_hidden_states: + encoder_states = encoder_states + (hidden_states, ) + if self.gradient_checkpointing and self.training: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs, output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(encoder_layer), + hidden_states, + attention_mask, + ) + else: + layer_outputs = encoder_layer( + hidden_states, + attention_mask, + output_attentions=output_attentions, + ) + + hidden_states = layer_outputs[0] + + if output_attentions: + all_attentions = all_attentions + (layer_outputs[1], ) + + if output_hidden_states: + encoder_states = encoder_states + (hidden_states, ) + + if not return_dict: + return tuple( + v for v in [hidden_states, encoder_states, all_attentions] + if v is not None) + return BaseModelOutput( + last_hidden_state=hidden_states, + hidden_states=encoder_states, + attentions=all_attentions) + + +class MplugOwlVisionModel(MplugOwlPreTrainedModel): + main_input_name = 'pixel_values' + config_class = MplugOwlVisionConfig + + def __init__(self, config: MplugOwlVisionConfig): + super().__init__(config) + self.config = config + self.hidden_size = config.hidden_size + + self.embeddings = MplugOwlVisionEmbeddings(config) + self.encoder = MplugOwlVisionEncoder(config) + self.post_layernorm = LayerNormFp32( + self.hidden_size, eps=config.layer_norm_eps) + + self.post_init() + + def forward( + self, + pixel_values: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + Returns: + + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if pixel_values is None: + raise ValueError('You have to specify pixel_values') + + hidden_states = self.embeddings(pixel_values) + + encoder_outputs = self.encoder( + inputs_embeds=hidden_states, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + last_hidden_state = encoder_outputs[0] + last_hidden_state = self.post_layernorm(last_hidden_state) + + pooled_output = last_hidden_state[:, 0, :] + pooled_output = self.post_layernorm(pooled_output) + + if not return_dict: + return (last_hidden_state, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) + + def get_input_embeddings(self): + return self.embeddings + + +class MplugOwlVisualAbstractorMLP(nn.Module): + + def __init__(self, config: MplugOwlVisualAbstractorConfig): + super().__init__() + self.config = config + in_features = config.hidden_size + hidden_features = config.intermediate_size + hidden_features = int(2 * hidden_features / 3) + multiple_of = 256 + hidden_features = multiple_of * \ + ((hidden_features + multiple_of - 1) // multiple_of) + self.act = nn.SiLU() + + self.w1 = nn.Linear(in_features, hidden_features) + self.w2 = nn.Linear(hidden_features, in_features) + self.w3 = nn.Linear(in_features, hidden_features) + self.ffn_ln = LayerNormFp32(hidden_features, eps=config.layer_norm_eps) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.act( + self.w1(hidden_states)) * self.w3(hidden_states) + hidden_states = self.ffn_ln(hidden_states) + hidden_states = self.w2(hidden_states) + return hidden_states + + +class MplugOwlVisualAbstractorMultiHeadAttention(nn.Module): + + def __init__(self, config: MplugOwlVisualAbstractorConfig): + super().__init__() + self.config = config + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + 'The hidden size (%d) is not a multiple of the number of attention heads (%d)' + % (config.hidden_size, config.num_attention_heads)) + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.encoder_hidden_size, self.all_head_size) + self.value = nn.Linear(config.encoder_hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.save_attention = False + + def save_attn_gradients(self, attn_gradients): + self.attn_gradients = attn_gradients + + def get_attn_gradients(self): + return self.attn_gradients + + def save_attention_map(self, attention_map): + self.attention_map = attention_map + + def get_attention_map(self): + return self.attention_map + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + key_layer = self.transpose_for_scores(self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores( + self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + + mixed_query_layer = self.query(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + + attention_scores = attention_scores / \ + math.sqrt(self.attention_head_size) + + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + if self.save_attention: + self.save_attention_map(attention_probs) + attention_probs.register_hook(self.save_attn_gradients) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs_dropped = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs_dropped = attention_probs_dropped * head_mask + + context_layer = torch.matmul(attention_probs_dropped, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, + attention_probs) if output_attentions else (context_layer, ) + + outputs = outputs + (past_key_value, ) + return outputs + + +class MplugOwlVisualAbstractorCrossOutput(nn.Module): + + def __init__(self, config: MplugOwlVisualAbstractorConfig): + super().__init__() + dim = config.hidden_size + self.out_proj = nn.Linear(dim, dim, bias=True) + self.norm2 = LayerNormFp32(dim) + self.mlp = MplugOwlVisualAbstractorMLP(config) + + def forward(self, hidden_states: torch.Tensor, + input_tensor: torch.Tensor) -> torch.Tensor: + input_tensor = input_tensor + self.out_proj(hidden_states) + input_tensor = input_tensor + self.mlp(self.norm2(input_tensor)) + return input_tensor + + +class MplugOwlVisualAbstractorAttention(nn.Module): + + def __init__(self, config: MplugOwlVisualAbstractorConfig): + super().__init__() + self.attention = MplugOwlVisualAbstractorMultiHeadAttention(config) + self.output = MplugOwlVisualAbstractorCrossOutput(config) + self.pruned_heads = set() + self.norm1 = LayerNormFp32(config.hidden_size) + self.normk = LayerNormFp32(config.hidden_size) + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.attention.num_attention_heads, + self.attention.attention_head_size, self.pruned_heads) + + # Prune linear layers + self.attention.query = prune_linear_layer(self.attention.query, index) + self.attention.key = prune_linear_layer(self.attention.key, index) + self.attention.value = prune_linear_layer(self.attention.value, index) + self.output.dense = prune_linear_layer( + self.output.out_proj, index, dim=1) + + # Update hyper params and store pruned heads + self.attention.num_attention_heads = self.attention.num_attention_heads - \ + len(heads) + self.attention.all_head_size = self.attention.attention_head_size * \ + self.attention.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor]: + # HACK we apply norm on q and k + hidden_states = self.norm1(hidden_states) + encoder_hidden_states = self.normk(encoder_hidden_states) + encoder_hidden_states = torch.cat( + [hidden_states, encoder_hidden_states], dim=1) + encoder_attention_mask = torch.cat( + [attention_mask, encoder_attention_mask], dim=-1) + self_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + attention_output = self.output(self_outputs[0], hidden_states) + # add attentions if we output them + outputs = (attention_output, ) + self_outputs[1:] + return outputs + + +class MplugOwlVisualAbstractorLayer(nn.Module): + + def __init__(self, config, layer_idx): + super().__init__() + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + + self.layer_idx = layer_idx + + self.crossattention = MplugOwlVisualAbstractorAttention(config) + self.has_cross_attention = True + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + output_attentions=False, + ): + if encoder_hidden_states is None: + raise ValueError( + 'encoder_hidden_states must be given for cross-attention layers' + ) + cross_attention_outputs = self.crossattention( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + output_attentions=output_attentions, + ) + query_attention_output = cross_attention_outputs[0] + + outputs = (query_attention_output, ) + return outputs + + +class MplugOwlVisualAbstractorEncoder(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.layers = nn.ModuleList([ + MplugOwlVisualAbstractorLayer(config, layer_idx) + for layer_idx in range(config.num_hidden_layers) + ]) + self.gradient_checkpointing = False + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + ): + all_hidden_states = () if output_hidden_states else None + + for i in range(self.config.num_hidden_layers): + layer_module = self.layers[i] + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[ + i] if past_key_values is not None else None + + if getattr(self.config, 'gradient_checkpointing', + False) and self.training: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs, past_key_value, + output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + output_attentions, + ) + + hidden_states = layer_outputs[0] + + return BaseModelOutput(last_hidden_state=hidden_states, ) + + +class MplugOwlVisualAbstractorModel(MplugOwlPreTrainedModel): + + def __init__(self, config: MplugOwlVisualAbstractorConfig, + language_hidden_size): + super().__init__(config) + self.config = config + + self.encoder = MplugOwlVisualAbstractorEncoder(config) + self.visual_fc = torch.nn.Linear(config.hidden_size, + language_hidden_size) + self.vit_eos = torch.nn.Parameter( + torch.randn(1, 1, language_hidden_size)) + self.post_init() + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def get_extended_attention_mask( + self, + attention_mask: torch.Tensor, + input_shape: Tuple[int], + device: torch.device, + ) -> torch.Tensor: + """ + Makes broadcastable attention and causal masks so that future and masked tokens are ignored. + + Arguments: + attention_mask (`torch.Tensor`): + Mask with ones indicating tokens to attend to, zeros for tokens to ignore. + input_shape (`Tuple[int]`): + The shape of the input to the model. + device: (`torch.device`): + The device of the input to the model. + + Returns: + `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`. + """ + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + if attention_mask.dim() == 3: + extended_attention_mask = attention_mask[:, None, :, :] + elif attention_mask.dim() == 2: + extended_attention_mask = attention_mask[:, None, None, :] + else: + raise ValueError( + 'Wrong shape for input_ids (shape {}) or attention_mask (shape {})' + .format(input_shape, attention_mask.shape)) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to( + dtype=self.dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + return extended_attention_mask + + def forward( + self, + query_embeds, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, `optional`): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors: + shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and + value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are + used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key + value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape + `(batch_size, sequence_length)`. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + embedding_output = query_embeds + input_shape = embedding_output.size()[:-1] + batch_size, seq_length = input_shape + device = embedding_output.device + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + if attention_mask is None: + attention_mask = torch.ones( + (query_embeds.shape[0], query_embeds.shape[1]), + dtype=torch.long, + device=query_embeds.device) + extended_attention_mask = self.get_extended_attention_mask( + attention_mask, input_shape, device) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if encoder_hidden_states is not None: + if type(encoder_hidden_states) == list: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states[ + 0].size() + else: + ( + encoder_batch_size, + encoder_sequence_length, + _, + ) = encoder_hidden_states.size() + encoder_hidden_shape = (encoder_batch_size, + encoder_sequence_length) + + if type(encoder_attention_mask) == list: + encoder_extended_attention_mask = [ + self.invert_attention_mask(mask) + for mask in encoder_attention_mask + ] + elif encoder_attention_mask is None: + encoder_attention_mask = torch.ones( + encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, + self.config.num_hidden_layers) + + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = encoder_outputs[0] + pooled_output = sequence_output[:, 0, :] + + sequence_output = self.visual_fc(sequence_output) + eos_repeat = self.vit_eos.repeat(sequence_output.shape[0], 1, 1) + sequence_output = torch.cat([sequence_output, eos_repeat], dim=1) + + return BaseModelOutputWithPooling( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + ) + + +class MplugOwlModel(MplugOwlPreTrainedModel): + r"""The mPLUG-Owl model is a multi-modal conversation model that support various modalities as input. + mPLUG-Owl consists a visual encoder, a visual abstrator module and a language decoder model, which enables + both image and text input. + This model is implemented base on mPLUG-Owl: Modularization Empowers Large Language Models with Multimodality. + `Paper `. + """ + config_class = MplugOwlConfig + main_input_name = 'pixel_values' + + def __init__(self, config: MplugOwlConfig): + super().__init__(config) + + self.vision_model = MplugOwlVisionModel(config.vision_config) + + self.query_tokens = nn.Parameter( + torch.zeros(1, config.num_query_tokens, + config.visual_abstractor_config.hidden_size)) + self.abstractor = MplugOwlVisualAbstractorModel( + config.visual_abstractor_config, config.text_config.hidden_size) + + # if config.use_decoder_only_language_model: + language_model = AutoModelForCausalLM.from_config(config.text_config) + self.language_model = language_model + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.language_model.get_input_embeddings() + + def set_input_embeddings(self, value): + self.language_model.set_input_embeddings(value) + + def set_output_embeddings(self, new_embeddings): + self.language_model.set_output_embeddings(new_embeddings) + + def get_output_embeddings(self) -> nn.Module: + return self.language_model.get_output_embeddings() + + def get_encoder(self): + return self.language_model.get_encoder() + + def get_decoder(self): + return self.language_model.get_decoder() + + def _tie_weights(self): + if not self.config.use_decoder_only_language_model: + self.language_model.encoder.embed_tokens = self.language_model.shared + self.language_model.decoder.embed_tokens = self.language_model.shared + + def get_text_features( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + decoder_input_ids: Optional[torch.Tensor] = None, + decoder_attention_mask: Optional[torch.Tensor] = None, + labels: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ): + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if self.config.use_decoder_only_language_model: + text_outputs = self.language_model( + input_ids=input_ids, + attention_mask=attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + else: + inputs_embeds = self.language_model.get_input_embeddings()( + input_ids) + + text_outputs = self.language_model( + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + labels=labels, + ) + + return text_outputs + + def get_image_features( + self, + pixel_values: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ): + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + vision_outputs = self.vision_model( + pixel_values=pixel_values, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + return vision_outputs + + +def get_media_indices(my_list): + if isinstance(my_list, torch.Tensor): + my_list = my_list.cpu().tolist() + result = [] + for i in range(len(my_list)): + if i == 0 and my_list[i] < 0: + result.append(i) + elif my_list[i] != my_list[i - 1] and my_list[i] < 0: + result.append(i) + return result + + +class MplugOwlForConditionalGenerationHF(MplugOwlPreTrainedModel): + config_class = MplugOwlConfig + main_input_name = 'pixel_values' + + def __init__(self, config: MplugOwlConfig, **kwargs): + super().__init__(config) + + self.vision_model = MplugOwlVisionModel(config.vision_config) + + self.query_tokens = nn.Parameter( + torch.zeros(1, config.num_query_tokens, + config.visual_abstractor_config.hidden_size)) + self.abstractor = MplugOwlVisualAbstractorModel( + config.visual_abstractor_config, config.text_config.hidden_size) + + # if config.use_decoder_only_language_model: + language_model = AutoModelForCausalLM.from_config(config.text_config) + self.language_model = language_model + + # Initialize weights and apply final processing + self.post_init() + self.main_input_name = 'input_ids' + + def get_input_embeddings(self): + return self.language_model.get_input_embeddings() + + def set_input_embeddings(self, value): + self.language_model.set_input_embeddings(value) + + def set_output_embeddings(self, new_embeddings): + self.language_model.set_output_embeddings(new_embeddings) + + def get_output_embeddings(self) -> nn.Module: + return self.language_model.get_output_embeddings() + + def get_encoder(self): + return self.language_model.get_encoder() + + def get_decoder(self): + return self.language_model.get_decoder() + + def _tie_weights(self): + if not self.config.use_decoder_only_language_model: + self.language_model.encoder.embed_tokens = self.language_model.shared + self.language_model.decoder.embed_tokens = self.language_model.shared + + def _preprocess_accelerate(self): + r""" + Some pre-processing hacks to make the model `accelerate` compatible. Check + https://github.com/huggingface/transformers/pull/21707 for more details. + """ + hf_device_map = self.hf_device_map + + if len( + hf_device_map + ) > 1 and 'language_model' not in hf_device_map and torch.cuda.device_count( + ) > 1: + # warn users about unexpected behavior when using multi-GPU + mPLUG-Owl + `accelerate`. + logger.warning( + 'The `language_model` is not in the `hf_device_map` dictionary and you are running your script' + ' in a multi-GPU environment. this may lead to unexpected behavior when using `accelerate`.' + ' Please pass a `device_map` that contains `language_model` to remove this warning.' + ' Please refer to https://github.com/huggingface/blog/blob/main/accelerate-large-models.md for' + ' more details on creating a `device_map` for large models.', ) + + if hasattr(self.language_model, '_hf_hook'): + self.language_model._hf_hook.io_same_device = True # For `generate` compatibility + + def forward( + self, + pixel_values: torch.FloatTensor, + input_ids: torch.FloatTensor, + num_images, + non_padding_mask: Optional[torch.LongTensor] = None, + non_media_mask: Optional[torch.LongTensor] = None, + prompt_mask: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + labels: Optional[torch.LongTensor] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, MplugOwlForConditionalGenerationModelOutput]: + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # get text embedding + text_tokens_ = input_ids + batch_size = input_ids.shape[0] + + media_token_indices = [ + # [:-1] since we would not use the last token for embedding + get_media_indices(text_tokens_[i][:-1]) for i in range(batch_size) + ] + text_tokens_[text_tokens_ < 0] = 1 # Not used + text_embeds = self.get_input_embeddings()( + text_tokens_) # Temporally Embedding + + if pixel_values is not None: + pixel_values = pixel_values.half() + image_embeds = self.vision_model( + pixel_values, return_dict=True).last_hidden_state + + image_attention_mask = torch.ones( + image_embeds.size()[:-1], + dtype=torch.long, + device=image_embeds.device) + query_tokens = self.query_tokens.expand(image_embeds.shape[0], -1, + -1) + + query_features = self.abstractor( + query_embeds=query_tokens, + encoder_hidden_states=image_embeds, + encoder_attention_mask=image_attention_mask, + )['last_hidden_state'] + img_seq_length = query_features.shape[1] + + num_images_per_sample = num_images.long().cpu().tolist() + + text_chunk_embeds = [] + img_idx = 0 + for b in range(batch_size): + start = 0 + result = [] + if len(media_token_indices[b]) > 0: + for i, pos in enumerate(media_token_indices[b]): + if pos > start: + result.append(text_embeds[b, start:pos]) + result.append(query_features[img_idx + i]) + start = pos + img_seq_length + if start < text_embeds.shape[1]: + result.append(text_embeds[b, start:]) + + img_idx += num_images_per_sample[b] + text_chunk_embeds.append(torch.cat(result, dim=0)) + + # Actual Input Embeddings + input_embeds = torch.stack(text_chunk_embeds, dim=0) + + # Create causal mask and position ids + _, loss_mask, position_ids = \ + get_ltor_masks_and_position_ids_from_embeddings(input_embeds) + + # Calculate the loss_mask + non_padding_mask = non_padding_mask.long() + non_media_mask = non_media_mask.long() + prompt_mask = prompt_mask.long() # TODO How to deal with prompt mask + loss_mask = loss_mask[:, :-1] + + loss_mask = loss_mask * non_padding_mask * non_media_mask * prompt_mask + + # Forward into GPT + outputs = self.language_model( + inputs_embeds=input_embeds, + attention_mask=attention_mask, + labels=labels, + ) + outputs.loss = (outputs.loss + * loss_mask.view(-1)).sum() / loss_mask.sum() + return outputs + + @torch.no_grad() + def generate( + self, + pixel_values: torch.FloatTensor, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + **generate_kwargs, + ) -> torch.LongTensor: + """ + Overrides `generate` function to be able to use the model as a conditional generator. + + Args: + pixel_values (`torch.FloatTensor` of shape (batch_size, num_channels, height, width)): + Input images to be processed. + input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): + The sequence used as a prompt for the generation. + attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): + Mask to avoid performing attention on padding token indices + + Returns: + captions (list): A list of strings of length batch_size * num_captions. + """ + + if input_ids is not None: + batch_size = input_ids.size(0) + media_token_indices = [ + get_media_indices(input_ids[i]) for i in range(batch_size) + ] + num_images_per_sample = [len(x) for x in media_token_indices] + input_ids = input_ids.clone() + input_ids[input_ids < 0] = 0 # Not used + + if attention_mask is None: + attention_mask = torch.ones_like(input_ids).long().to( + input_ids.device) + + if hasattr(self, 'hf_device_map'): + # preprocess for `accelerate` + self._preprocess_accelerate() + batch_size = input_ids.shape[0] + # get text embedding + inputs_embeds = self.get_input_embeddings()(input_ids) + # get visual embedding + if pixel_values is not None: + pixel_values = pixel_values.half() + pixel_values = pixel_values.to(input_ids.device) + with torch.no_grad(): + image_embeds = self.vision_model( + pixel_values, return_dict=True).last_hidden_state + image_attention_mask = torch.ones( + image_embeds.size()[:-1], + dtype=torch.long, + device=image_embeds.device) + query_tokens = self.query_tokens.expand( + image_embeds.shape[0], -1, -1) + query_outputs = self.abstractor( + query_embeds=query_tokens, + encoder_hidden_states=image_embeds, + encoder_attention_mask=image_attention_mask, + return_dict=True, + ) + query_output = query_outputs['last_hidden_state'] + image_embeds = query_output + img_seq_length = image_embeds.shape[1] + + # =================== + # Get actual input embeddings + # =================== + text_chunk_embeds = [] + text_chunk_attns = [] + img_idx = 0 + + for b in range(batch_size): + start = 0 + result = [] + result_attn = [] + for i, pos in enumerate(media_token_indices[b]): + if pos > start: + result.append(inputs_embeds[b, start:pos]) + result_attn.append(attention_mask[b, start:pos]) + result.append(image_embeds[img_idx + i]) + result_attn.append( + torch.ones( + image_embeds[img_idx + i].shape[0], + device=inputs_embeds.device)) + start = pos + img_seq_length + if start < inputs_embeds.shape[1]: + result.append(inputs_embeds[b, start:]) + result_attn.append(attention_mask[b, start:]) + + img_idx += num_images_per_sample[b] + text_chunk_embeds.append(torch.cat(result, dim=0)) + text_chunk_attns.append(torch.cat(result_attn, dim=0)) + inputs_embeds = torch.stack(text_chunk_embeds, dim=0) + attention_mask = torch.stack(text_chunk_attns, dim=0) + + outputs = self.language_model.generate( + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + **generate_kwargs, + ) + + return outputs + + +@MODELS.register_module( + Tasks.multimodal_dialogue, module_name=Models.mplug_owl) +class MplugOwlForConditionalGeneration(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the mPLUG-Owl model from the `model_dir` path. + Args: + model_dir (str): the model path. + """ + + super().__init__(model_dir, *args, **kwargs) + self.model = MplugOwlForConditionalGenerationHF.from_pretrained( + model_dir, + torch_dtype=torch.half, + ) + + def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + output = self.model.generate(**input) + return output diff --git a/modelscope/outputs/outputs.py b/modelscope/outputs/outputs.py index 1b06795a..ab24a34c 100644 --- a/modelscope/outputs/outputs.py +++ b/modelscope/outputs/outputs.py @@ -1369,6 +1369,10 @@ TASK_OUTPUTS = { # {"text": "this is a text answser. "} Tasks.video_question_answering: [OutputKeys.TEXT], + # Multimodal Dialogue result for a sample + # {"text": "this is a text response. "} + Tasks.multimodal_dialogue: [OutputKeys.TEXT], + # auto_speech_recognition result for a single sample # { # "text": "每天都要快乐喔" diff --git a/modelscope/pipeline_inputs.py b/modelscope/pipeline_inputs.py index 29c5b8d8..8cb031e7 100644 --- a/modelscope/pipeline_inputs.py +++ b/modelscope/pipeline_inputs.py @@ -337,6 +337,9 @@ TASK_INPUTS = { Tasks.video_captioning: [InputType.VIDEO, { 'video': InputType.VIDEO, }], + Tasks.multimodal_dialogue: { + 'messages': InputType.LIST, + }, Tasks.visual_grounding: { 'image': InputType.IMAGE, 'text': InputType.TEXT diff --git a/modelscope/pipelines/multi_modal/__init__.py b/modelscope/pipelines/multi_modal/__init__.py index 2e496952..b28e9a71 100644 --- a/modelscope/pipelines/multi_modal/__init__.py +++ b/modelscope/pipelines/multi_modal/__init__.py @@ -21,6 +21,7 @@ if TYPE_CHECKING: from .diffusers_wrapped import StableDiffusionWrapperPipeline, ChineseStableDiffusionPipeline from .soonet_video_temporal_grounding_pipeline import SOONetVideoTemporalGroundingPipeline from .text_to_video_synthesis_pipeline import TextToVideoSynthesisPipeline + from .multimodal_dialogue_pipeline import MultimodalDialoguePipeline else: _import_structure = { 'image_captioning_pipeline': ['ImageCaptioningPipeline'], @@ -45,6 +46,7 @@ else: 'soonet_video_temporal_grounding_pipeline': ['SOONetVideoTemporalGroundingPipeline'], 'text_to_video_synthesis_pipeline': ['TextToVideoSynthesisPipeline'], + 'multimodal_dialogue_pipeline': ['MultimodalDialoguePipeline'] } import sys diff --git a/modelscope/pipelines/multi_modal/multimodal_dialogue_pipeline.py b/modelscope/pipelines/multi_modal/multimodal_dialogue_pipeline.py new file mode 100644 index 00000000..31df19fc --- /dev/null +++ b/modelscope/pipelines/multi_modal/multimodal_dialogue_pipeline.py @@ -0,0 +1,90 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Any, Dict, Optional, Union + +import torch + +from modelscope.metainfo import Pipelines +from modelscope.models.multi_modal import MplugOwlForConditionalGeneration +from modelscope.outputs import OutputKeys, TokenGeneratorOutput +from modelscope.pipelines.base import Model, Pipeline +from modelscope.pipelines.builder import PIPELINES +from modelscope.preprocessors import MplugOwlPreprocessor, Preprocessor +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@PIPELINES.register_module( + Tasks.multimodal_dialogue, module_name=Pipelines.multimodal_dialogue) +class MultimodalDialoguePipeline(Pipeline): + r""" Multimodal Dialogue Pipeline. + + Examples: + >>> from modelscope.pipelines import pipeline + >>> chatbot = pipeline('multimodal-dialogue', 'damo/multi-modal_mplug_owl_multimodal-dialogue_7b') + >>> image = 'data/resource/portrait_input.png' + >>> system_prompt_1 = 'The following is a conversation between a curious human and AI assistant.' + >>> system_prompt_2 = "The assistant gives helpful, detailed, and polite answers to the user's questions." + >>> messages = { + >>> 'messages': [ + >>> { + >>> 'role': 'system', + >>> 'content': system_prompt_1 + ' ' + system_prompt_2 + >>> }, + >>> { + >>> 'role': 'user', + >>> 'content': [{ + >>> 'image': image + >>> }] + >>> }, + >>> { + >>> 'role': 'user', + >>> 'content': 'Describe the facial expression of the man.' + >>> }, + >>> ] + >>> } + >>> chatbot(messages) + >>> { + >>> "text": he is angry. + >>> } + >>> + """ + + def __init__(self, + model: Union[Model, str], + preprocessor: Optional[Preprocessor] = None, + **kwargs): + """ + use `model` and `preprocessor` to create a multimodal dialogue pipeline for prediction + Args: + model: model id on modelscope hub. + """ + super().__init__(model=model, preprocessor=preprocessor, **kwargs) + self.model.eval() + if preprocessor is None: + if isinstance(self.model, MplugOwlForConditionalGeneration): + self.preprocessor = MplugOwlPreprocessor(self.model.model_dir) + + def forward(self, inputs: Dict[str, Any], + **forward_params) -> Dict[str, Any]: + """ + the `forward_params` can be the generation configurations listed in transformers library. + """ + with torch.no_grad(): + return super().forward(inputs, **forward_params) + + def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, str]: + """process the prediction results + + Args: + inputs (Dict[str, Any]): _description_ + + Returns: + Dict[str, str]: the prediction results + """ + if isinstance(self.model, MplugOwlForConditionalGeneration): + output = self.preprocessor.tokenizer.decode( + inputs[0], skip_special_tokens=True) + inputs = {OutputKeys.TEXT: output} + return inputs diff --git a/modelscope/preprocessors/__init__.py b/modelscope/preprocessors/__init__.py index 3bbca124..dbcb0813 100644 --- a/modelscope/preprocessors/__init__.py +++ b/modelscope/preprocessors/__init__.py @@ -20,7 +20,7 @@ if TYPE_CHECKING: from .tts import KanttsDataPreprocessor from .multi_modal import (DiffusionImageGenerationPreprocessor, OfaPreprocessor, MPlugPreprocessor, - HiTeAPreprocessor, + HiTeAPreprocessor, MplugOwlPreprocessor, ImageCaptioningClipInterrogatorPreprocessor) from .nlp import ( DocumentSegmentationTransformersPreprocessor, @@ -71,7 +71,7 @@ else: 'tts': ['KanttsDataPreprocessor'], 'multi_modal': [ 'DiffusionImageGenerationPreprocessor', 'OfaPreprocessor', - 'MPlugPreprocessor', 'HiTeAPreprocessor', + 'MPlugPreprocessor', 'HiTeAPreprocessor', 'MplugOwlPreprocessor', 'ImageCaptioningClipInterrogatorPreprocessor' ], 'nlp': [ diff --git a/modelscope/preprocessors/multi_modal.py b/modelscope/preprocessors/multi_modal.py index eb7ae339..faf796f4 100644 --- a/modelscope/preprocessors/multi_modal.py +++ b/modelscope/preprocessors/multi_modal.py @@ -1,5 +1,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import os.path as osp +import re from io import BytesIO from typing import Any, Dict, List, Tuple, Union @@ -29,7 +30,7 @@ from .ofa.utils.constant import OFA_TASK_KEY_MAPPING __all__ = [ 'DiffusionImageGenerationPreprocessor', 'OfaPreprocessor', - 'MPlugPreprocessor', 'HiTeAPreprocessor' + 'MPlugPreprocessor', 'HiTeAPreprocessor', 'MplugOwlPreprocessor' ] @@ -644,6 +645,148 @@ class HiTeAPreprocessor(Preprocessor): return output +@PREPROCESSORS.register_module( + Fields.multi_modal, module_name=Preprocessors.mplug_owl_preprocessor) +class MplugOwlPreprocessor(Preprocessor): + + def __init__(self, + model_dir: str, + mode: str = ModeKeys.INFERENCE, + *args, + **kwargs): + super().__init__(*args, **kwargs) + self.model_dir = model_dir + self.mode = mode + + self._tokenizer = None + self._patch_resize_transform = None + self.media_token = {'': 65} + self._image_map = {} + + @property + def tokenizer(self): + from modelscope.models.nlp.llama import LlamaTokenizer + + if self._tokenizer is None: + self._tokenizer = LlamaTokenizer.from_pretrained(self.model_dir) + return self._tokenizer + + @property + def patch_resize_transform(self): + if self._patch_resize_transform is None: + from torchvision import transforms + + mean = (0.48145466, 0.4578275, 0.40821073) + std = (0.26862954, 0.26130258, 0.27577711) + + self._patch_resize_transform = transforms.Compose([ + transforms.Resize((224, 224), interpolation=Image.BICUBIC), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std), + ]) + return self._patch_resize_transform + + def image_open(self, path: str) -> Tuple[Image.Image, int]: + if path not in self._image_map: + index = len(self._image_map) + self._image_map[path] = (load_image(path), index) + return self._image_map[path] + + def tokenize_text(self, text: str) -> List[int]: + media_tokens = { + k: -int(i + 1) + for i, k in enumerate(self.media_token.keys()) + } + media_lengths = self.media_token.copy() + + prompt_chunk = [self.tokenizer.bos_token_id] + + # Pure Text + condition = [ + media_token not in text for media_token in media_tokens.keys() + ] + if all(condition): + enc_chunk = prompt_chunk + \ + self.tokenizer(text, add_special_tokens=False)['input_ids'] + + # Multi-Modal Text + else: + enc_chunk = prompt_chunk + pattern = '|'.join(map(re.escape, list(media_tokens.keys()))) + chunk_strs = re.split(f'({pattern})', text) + chunk_strs = [x for x in chunk_strs if len(x) > 0] + for idx, chunk_str in enumerate(chunk_strs): + if chunk_str in media_tokens: + enc_chunk += [media_tokens[chunk_str]] * \ + media_lengths[chunk_str] + else: + tmp_chunk = self.tokenizer( + chunk_str, add_special_tokens=False)['input_ids'] + enc_chunk += tmp_chunk + return enc_chunk + + def convert(self, messages: Dict[str, List[Dict]]) -> str: + texts = [] + image = [] + messages = messages['messages'] + for turn in messages: + if turn['role'] == 'system': + role = '' + elif turn['role'] == 'user': + role = 'Human: ' + else: + role = 'AI: ' + if isinstance(turn['content'], str): + text = f"{role}{turn['content']}" + texts.append(text) + else: + for t in turn['content']: + if isinstance(t, str): + text = f'{role}{t}' + else: + text = f'{role}' + image.append(t['image']) + texts.append(text) + texts = '\n'.join(texts) + texts += '\nAI: ' + return image, texts + + def __call__(self, messages: Dict[str, Any]) -> Dict[str, Any]: + """ + Args: + messages: {[ + {'role': 'system', 'content': 'message1'}, + {'role': 'user', 'content': 'message2'}, + {'role': 'user', 'content': ['message2', {"image": 'image_path'}, 'message3', ...]}, + ]} + The 'role' should be choose from ['system', 'user', 'assistant']. + The 'content' can be either str or List[Union[str, Dict]] + Return: + output: Dict[str, Tensor] + """ + output = {} + images, text = self.convert(messages) + + if len(images) > 0: + pixel_values = [] + for image in images: + pixel_values.append( + self.patch_resize_transform(self.image_open(image)[0])) + pixel_values = torch.stack(pixel_values, dim=0) + else: + pixel_values = None + + input_ids = self.tokenize_text(text) + input_ids = torch.LongTensor([input_ids]) + + output = { + 'pixel_values': pixel_values, + 'input_ids': input_ids, + } + + return output + + @PREPROCESSORS.register_module( Fields.multi_modal, module_name=Preprocessors.image_captioning_clip_interrogator_preprocessor) diff --git a/modelscope/utils/constant.py b/modelscope/utils/constant.py index 929369ec..1f44fc01 100644 --- a/modelscope/utils/constant.py +++ b/modelscope/utils/constant.py @@ -247,6 +247,7 @@ class MultiModalTasks(object): video_temporal_grounding = 'video-temporal-grounding' text_to_video_synthesis = 'text-to-video-synthesis' efficient_diffusion_tuning = 'efficient-diffusion-tuning' + multimodal_dialogue = 'multimodal-dialogue' class ScienceTasks(object): diff --git a/tests/pipelines/test_clip_interrogator.py b/tests/pipelines/test_clip_interrogator.py index 960db86d..615aef3c 100644 --- a/tests/pipelines/test_clip_interrogator.py +++ b/tests/pipelines/test_clip_interrogator.py @@ -1,17 +1,14 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import unittest -from PIL import Image - from modelscope.models import Model from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class CLIPInterrogatorTest(unittest.TestCase, DemoCompatibilityCheck): +class CLIPInterrogatorTest(unittest.TestCase): @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_run_with_image_captioning_with_model(self): @@ -32,10 +29,6 @@ class CLIPInterrogatorTest(unittest.TestCase, DemoCompatibilityCheck): result = pipeline_caption(image) print(result[OutputKeys.CAPTION]) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_face_recognition_onnx_transface.py b/tests/pipelines/test_face_recognition_onnx_transface.py index a41271c1..183257f0 100644 --- a/tests/pipelines/test_face_recognition_onnx_transface.py +++ b/tests/pipelines/test_face_recognition_onnx_transface.py @@ -6,11 +6,10 @@ import numpy as np from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class TransFaceRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): +class TransFaceRecognitionTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.face_recognition @@ -31,10 +30,6 @@ class TransFaceRecognitionTest(unittest.TestCase, DemoCompatibilityCheck): sim = np.dot(emb1[0], emb2[0]) print(f'Cos similarity={sim:.3f}, img1:{img1} img2:{img2}') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_fast_instance_segmentation.py b/tests/pipelines/test_fast_instance_segmentation.py index aefd1092..d5789150 100644 --- a/tests/pipelines/test_fast_instance_segmentation.py +++ b/tests/pipelines/test_fast_instance_segmentation.py @@ -5,11 +5,10 @@ from modelscope.models import Model from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks -from modelscope.utils.demo_utils import DemoCompatibilityCheck from modelscope.utils.test_utils import test_level -class FastInstanceSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): +class FastInstanceSegmentationTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.image_segmentation @@ -30,10 +29,6 @@ class FastInstanceSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): task=Tasks.image_segmentation, model=model, preprocessor=None) print(pipeline_parsing(input=self.image)[OutputKeys.LABELS]) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() diff --git a/tests/pipelines/test_mplug_owl_multimodal_dialogue.py b/tests/pipelines/test_mplug_owl_multimodal_dialogue.py new file mode 100644 index 00000000..57bce67e --- /dev/null +++ b/tests/pipelines/test_mplug_owl_multimodal_dialogue.py @@ -0,0 +1,100 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import unittest + +from PIL import Image + +from modelscope.models import Model +from modelscope.outputs import OutputKeys +from modelscope.pipelines import pipeline +from modelscope.utils.constant import Tasks +from modelscope.utils.test_utils import test_level + + +class MplugOwlMultimodalDialogueTest(unittest.TestCase): + + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') + def test_run_with_multimodal_dialogue_with_model(self): + model = Model.from_pretrained( + 'damo/multi-modal_mplug_owl_multimodal-dialogue_7b') + pipeline_multimodal_dialogue = pipeline( + task=Tasks.multimodal_dialogue, + model=model, + ) + image = 'data/resource/portrait_input.png' + system_prompt_1 = 'The following is a conversation between a curious human and AI assistant.' + system_prompt_2 = "The assistant gives helpful, detailed, and polite answers to the user's questions." + messages = { + 'messages': [ + { + 'role': 'system', + 'content': system_prompt_1 + ' ' + system_prompt_2 + }, + { + 'role': 'user', + 'content': [{ + 'image': image + }] + }, + { + 'role': 'user', + 'content': 'Describe the facial expression of the man.' + }, + ] + } + result = pipeline_multimodal_dialogue(messages) + print(result[OutputKeys.TEXT]) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_multimodal_dialogue_with_name(self): + pipeline_multimodal_dialogue = pipeline( + Tasks.multimodal_dialogue, + model='damo/multi-modal_mplug_owl_multimodal-dialogue_7b') + image = 'data/resource/portrait_input.png' + system_prompt_1 = 'The following is a conversation between a curious human and AI assistant.' + system_prompt_2 = "The assistant gives helpful, detailed, and polite answers to the user's questions." + messages = { + 'messages': [ + { + 'role': 'system', + 'content': system_prompt_1 + ' ' + system_prompt_2 + }, + { + 'role': 'user', + 'content': [{ + 'image': image + }] + }, + { + 'role': 'user', + 'content': 'Describe the facial expression of the man.' + }, + ] + } + result = pipeline_multimodal_dialogue(messages) + print(result[OutputKeys.TEXT]) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_multimodal_dialogue_with_text(self): + pipeline_multimodal_dialogue = pipeline( + Tasks.multimodal_dialogue, + model='damo/multi-modal_mplug_owl_multimodal-dialogue_7b') + system_prompt_1 = 'The following is a conversation between a curious human and AI assistant.' + system_prompt_2 = "The assistant gives helpful, detailed, and polite answers to the user's questions." + messages = { + 'messages': [ + { + 'role': 'system', + 'content': system_prompt_1 + ' ' + system_prompt_2 + }, + { + 'role': 'user', + 'content': 'Where is the captial of China?' + }, + ] + } + result = pipeline_multimodal_dialogue(messages) + print(result[OutputKeys.TEXT]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/pipelines/test_speaker_verification.py b/tests/pipelines/test_speaker_verification.py index 84a079d2..ae498163 100644 --- a/tests/pipelines/test_speaker_verification.py +++ b/tests/pipelines/test_speaker_verification.py @@ -1,6 +1,5 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -import os.path import unittest from typing import Any, Dict, List, Union @@ -81,10 +80,6 @@ class SpeakerVerificationTest(unittest.TestCase): print(result) self.assertTrue(OutputKeys.TEXT in result) - @unittest.skip('demo compatibility test is only enabled on a needed-basis') - def test_demo_compatibility(self): - self.compatibility_check() - if __name__ == '__main__': unittest.main() From e325862be8fe4ad25d2cb18cebb18285b6fd01d9 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Mon, 15 May 2023 17:46:32 +0800 Subject: [PATCH 39/76] add pop for base class parameters Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12622643 --- .../pipelines/nlp/dialog_intent_prediction_pipeline.py | 3 ++- .../nlp/document_grounded_dialog_generate_pipeline.py | 3 ++- .../nlp/document_grounded_dialog_rerank_pipeline.py | 3 ++- .../nlp/document_grounded_dialog_retrieval_pipeline.py | 3 ++- modelscope/pipelines/nlp/feature_extraction_pipeline.py | 3 ++- modelscope/pipelines/nlp/fill_mask_pipeline.py | 3 ++- .../pipelines/nlp/named_entity_recognition_pipeline.py | 3 ++- modelscope/pipelines/nlp/sentence_embedding_pipeline.py | 3 ++- modelscope/pipelines/nlp/siamese_uie_pipeline.py | 5 +++-- .../pipelines/nlp/table_question_answering_pipeline.py | 3 ++- modelscope/pipelines/nlp/text_generation_pipeline.py | 3 ++- modelscope/pipelines/nlp/text_ranking_pipeline.py | 3 ++- modelscope/pipelines/nlp/token_classification_pipeline.py | 4 +++- .../pipelines/nlp/user_satisfaction_estimation_pipeline.py | 3 ++- .../pipelines/nlp/zero_shot_classification_pipeline.py | 3 ++- 15 files changed, 32 insertions(+), 16 deletions(-) diff --git a/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py b/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py index fa7b23b8..a0e75638 100644 --- a/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py +++ b/modelscope/pipelines/nlp/dialog_intent_prediction_pipeline.py @@ -41,7 +41,8 @@ class DialogIntentPredictionPipeline(Pipeline): config_file=config_file, device=device, auto_collate=auto_collate, - **kwargs) + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) if preprocessor is None: self.preprocessor = DialogIntentPredictionPreprocessor( self.model.model_dir, **kwargs) diff --git a/modelscope/pipelines/nlp/document_grounded_dialog_generate_pipeline.py b/modelscope/pipelines/nlp/document_grounded_dialog_generate_pipeline.py index 8c773dfe..dfcd95e6 100644 --- a/modelscope/pipelines/nlp/document_grounded_dialog_generate_pipeline.py +++ b/modelscope/pipelines/nlp/document_grounded_dialog_generate_pipeline.py @@ -47,7 +47,8 @@ class DocumentGroundedDialogGeneratePipeline(Pipeline): config_file=config_file, device=device, auto_collate=auto_collate, - **kwargs) + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) if preprocessor is None: self.preprocessor = DocumentGroundedDialogGeneratePreprocessor( diff --git a/modelscope/pipelines/nlp/document_grounded_dialog_rerank_pipeline.py b/modelscope/pipelines/nlp/document_grounded_dialog_rerank_pipeline.py index 8fdef380..29993594 100644 --- a/modelscope/pipelines/nlp/document_grounded_dialog_rerank_pipeline.py +++ b/modelscope/pipelines/nlp/document_grounded_dialog_rerank_pipeline.py @@ -65,7 +65,8 @@ class DocumentGroundedDialogRerankPipeline(Pipeline): device=device, auto_collate=auto_collate, seed=seed, - **kwarg) + compile=kwarg.pop('compile', False), + compile_options=kwarg.pop('compile_options', {})) self.model = model self.preprocessor = preprocessor self.device = device diff --git a/modelscope/pipelines/nlp/document_grounded_dialog_retrieval_pipeline.py b/modelscope/pipelines/nlp/document_grounded_dialog_retrieval_pipeline.py index c3fb1a32..31890a73 100644 --- a/modelscope/pipelines/nlp/document_grounded_dialog_retrieval_pipeline.py +++ b/modelscope/pipelines/nlp/document_grounded_dialog_retrieval_pipeline.py @@ -56,7 +56,8 @@ class DocumentGroundedDialogRetrievalPipeline(Pipeline): config_file=config_file, device=device, auto_collate=auto_collate, - **kwargs) + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) if preprocessor is None: self.preprocessor = DocumentGroundedDialogRetrievalPreprocessor( diff --git a/modelscope/pipelines/nlp/feature_extraction_pipeline.py b/modelscope/pipelines/nlp/feature_extraction_pipeline.py index 0f6979ba..c82db03c 100644 --- a/modelscope/pipelines/nlp/feature_extraction_pipeline.py +++ b/modelscope/pipelines/nlp/feature_extraction_pipeline.py @@ -54,7 +54,8 @@ class FeatureExtractionPipeline(Pipeline): config_file=config_file, device=device, auto_collate=auto_collate, - **kwargs) + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/fill_mask_pipeline.py b/modelscope/pipelines/nlp/fill_mask_pipeline.py index 6bc7622f..7b034786 100644 --- a/modelscope/pipelines/nlp/fill_mask_pipeline.py +++ b/modelscope/pipelines/nlp/fill_mask_pipeline.py @@ -63,7 +63,8 @@ class FillMaskPipeline(Pipeline): config_file=config_file, device=device, auto_collate=auto_collate, - **kwargs) + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/named_entity_recognition_pipeline.py b/modelscope/pipelines/nlp/named_entity_recognition_pipeline.py index 2cf30037..d035802b 100644 --- a/modelscope/pipelines/nlp/named_entity_recognition_pipeline.py +++ b/modelscope/pipelines/nlp/named_entity_recognition_pipeline.py @@ -56,7 +56,8 @@ class NamedEntityRecognitionPipeline(TokenClassificationPipeline): config_file=config_file, device=device, auto_collate=auto_collate, - **kwargs) + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/sentence_embedding_pipeline.py b/modelscope/pipelines/nlp/sentence_embedding_pipeline.py index 4e01397d..9d5cc80f 100644 --- a/modelscope/pipelines/nlp/sentence_embedding_pipeline.py +++ b/modelscope/pipelines/nlp/sentence_embedding_pipeline.py @@ -43,7 +43,8 @@ class SentenceEmbeddingPipeline(Pipeline): config_file=config_file, device=device, auto_collate=auto_collate, - **kwargs) + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/siamese_uie_pipeline.py b/modelscope/pipelines/nlp/siamese_uie_pipeline.py index cdbd9119..d548d2e8 100644 --- a/modelscope/pipelines/nlp/siamese_uie_pipeline.py +++ b/modelscope/pipelines/nlp/siamese_uie_pipeline.py @@ -21,7 +21,7 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines.base import Pipeline from modelscope.pipelines.builder import PIPELINES from modelscope.preprocessors import Preprocessor, SiameseUiePreprocessor -from modelscope.utils.constant import Tasks +from modelscope.utils.constant import ModelFile, Tasks Input = Union[str, tuple, MsDataset, 'Image.Image', 'numpy.ndarray'] @@ -68,7 +68,8 @@ class SiameseUiePipeline(Pipeline): config_file=config_file, device=device, auto_collate=auto_collate, - **kwargs) + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/table_question_answering_pipeline.py b/modelscope/pipelines/nlp/table_question_answering_pipeline.py index e174b762..7c064f57 100644 --- a/modelscope/pipelines/nlp/table_question_answering_pipeline.py +++ b/modelscope/pipelines/nlp/table_question_answering_pipeline.py @@ -52,7 +52,8 @@ class TableQuestionAnsweringPipeline(Pipeline): config_file=config_file, device=device, auto_collate=auto_collate, - **kwargs) + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/text_generation_pipeline.py b/modelscope/pipelines/nlp/text_generation_pipeline.py index 2b851dc4..d1aa5ff6 100644 --- a/modelscope/pipelines/nlp/text_generation_pipeline.py +++ b/modelscope/pipelines/nlp/text_generation_pipeline.py @@ -59,7 +59,8 @@ class TextGenerationPipeline(Pipeline): config_file=config_file, device=device, auto_collate=auto_collate, - **kwargs) + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/text_ranking_pipeline.py b/modelscope/pipelines/nlp/text_ranking_pipeline.py index a42baaa2..7539634e 100644 --- a/modelscope/pipelines/nlp/text_ranking_pipeline.py +++ b/modelscope/pipelines/nlp/text_ranking_pipeline.py @@ -44,7 +44,8 @@ class TextRankingPipeline(Pipeline): config_file=config_file, device=device, auto_collate=auto_collate, - **kwargs) + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/token_classification_pipeline.py b/modelscope/pipelines/nlp/token_classification_pipeline.py index daa4823c..9fd8e325 100644 --- a/modelscope/pipelines/nlp/token_classification_pipeline.py +++ b/modelscope/pipelines/nlp/token_classification_pipeline.py @@ -51,7 +51,9 @@ class TokenClassificationPipeline(Pipeline): preprocessor=preprocessor, config_file=config_file, device=device, - auto_collate=auto_collate) + auto_collate=auto_collate, + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) assert isinstance(self.model, Model), \ f'please check whether model config exists in {ModelFile.CONFIGURATION}' diff --git a/modelscope/pipelines/nlp/user_satisfaction_estimation_pipeline.py b/modelscope/pipelines/nlp/user_satisfaction_estimation_pipeline.py index 76fcd7a8..197a941f 100644 --- a/modelscope/pipelines/nlp/user_satisfaction_estimation_pipeline.py +++ b/modelscope/pipelines/nlp/user_satisfaction_estimation_pipeline.py @@ -51,7 +51,8 @@ class UserSatisfactionEstimationPipeline(Pipeline): config_file=config_file, device=device, auto_collate=auto_collate, - **kwargs) + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) if hasattr(self.preprocessor, 'id2label'): self.id2label = self.preprocessor.id2label diff --git a/modelscope/pipelines/nlp/zero_shot_classification_pipeline.py b/modelscope/pipelines/nlp/zero_shot_classification_pipeline.py index 9cd27adc..18ba40c8 100644 --- a/modelscope/pipelines/nlp/zero_shot_classification_pipeline.py +++ b/modelscope/pipelines/nlp/zero_shot_classification_pipeline.py @@ -67,7 +67,8 @@ class ZeroShotClassificationPipeline(Pipeline): config_file=config_file, device=device, auto_collate=auto_collate, - **kwargs) + compile=kwargs.pop('compile', False), + compile_options=kwargs.pop('compile_options', {})) self.entailment_id = 0 self.contradiction_id = 2 From 3ac31290f2d064bf43847083b6cef731321eee75 Mon Sep 17 00:00:00 2001 From: "wenmeng.zwm" Date: Mon, 15 May 2023 19:26:21 +0800 Subject: [PATCH 40/76] add description for modelscope --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9affe028..7ebb8dae 100644 --- a/setup.py +++ b/setup.py @@ -197,7 +197,8 @@ if __name__ == '__main__': setup( name='modelscope', version=get_version(), - description='', + description= + 'ModelScope: bring the notion of Model-as-a-Service to life.', long_description=readme(), long_description_content_type='text/markdown', author='Alibaba ModelScope team', From 6c8c3a53f8760e58199c63bac7d9c40438281c0f Mon Sep 17 00:00:00 2001 From: "zhifu.gzf" Date: Tue, 16 May 2023 13:49:01 +0800 Subject: [PATCH 41/76] fix bug in **kwargs duplicated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复**kwargs中参数重复的bug Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12633542 --- .../pipelines/audio/asr_inference_pipeline.py | 183 ++++++++---------- .../pipelines/audio/lm_infer_pipeline.py | 3 +- .../audio/punctuation_processing_pipeline.py | 3 +- .../audio/speaker_diarization_pipeline.py | 3 +- .../audio/speaker_verification_pipeline.py | 3 +- .../pipelines/audio/timestamp_pipeline.py | 3 +- .../voice_activity_detection_pipeline.py | 3 +- 7 files changed, 93 insertions(+), 108 deletions(-) diff --git a/modelscope/pipelines/audio/asr_inference_pipeline.py b/modelscope/pipelines/audio/asr_inference_pipeline.py index 09764c69..b9c0bd03 100644 --- a/modelscope/pipelines/audio/asr_inference_pipeline.py +++ b/modelscope/pipelines/audio/asr_inference_pipeline.py @@ -88,7 +88,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): beam_size('int'): beam size for decoding ctc_weight('float'): - CTC weight in joint decoding + the CTC weight in joint decoding lm_weight('float'): lm weight decoding_ind('int', defaults to 0): @@ -127,7 +127,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): minlenratio=self.cmd['minlenratio'], batch_size=self.cmd['batch_size'], beam_size=self.cmd['beam_size'], - ngpu=ngpu, + ngpu=self.cmd['ngpu'], ctc_weight=self.cmd['ctc_weight'], lm_weight=self.cmd['lm_weight'], penalty=self.cmd['penalty'], @@ -308,109 +308,88 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): } } - if self.framework == Frameworks.torch: - frontend_conf = None - token_num_relax = None - decoding_ind = None - decoding_mode = None - if os.path.exists(outputs['am_model_config']): - config_file = open( - outputs['am_model_config'], encoding='utf-8') - root = yaml.full_load(config_file) - config_file.close() - if 'frontend_conf' in root: - frontend_conf = root['frontend_conf'] - if os.path.exists(outputs['asr_model_config']): - config_file = open( - outputs['asr_model_config'], encoding='utf-8') - root = yaml.full_load(config_file) - config_file.close() - if 'token_num_relax' in root: - token_num_relax = root['token_num_relax'] - if 'decoding_ind' in root: - decoding_ind = root['decoding_ind'] - if 'decoding_mode' in root: - decoding_mode = root['decoding_mode'] + frontend_conf = None + token_num_relax = None + decoding_ind = None + decoding_mode = None + if os.path.exists(outputs['am_model_config']): + config_file = open(outputs['am_model_config'], encoding='utf-8') + root = yaml.full_load(config_file) + config_file.close() + if 'frontend_conf' in root: + frontend_conf = root['frontend_conf'] + if os.path.exists(outputs['asr_model_config']): + config_file = open(outputs['asr_model_config'], encoding='utf-8') + root = yaml.full_load(config_file) + config_file.close() + if 'token_num_relax' in root: + token_num_relax = root['token_num_relax'] + if 'decoding_ind' in root: + decoding_ind = root['decoding_ind'] + if 'decoding_mode' in root: + decoding_mode = root['decoding_mode'] - cmd['beam_size'] = root['beam_size'] - cmd['penalty'] = root['penalty'] - cmd['maxlenratio'] = root['maxlenratio'] - cmd['minlenratio'] = root['minlenratio'] - cmd['ctc_weight'] = root['ctc_weight'] - cmd['lm_weight'] = root['lm_weight'] - cmd['asr_train_config'] = outputs['am_model_config'] - cmd['lm_file'] = outputs['lm_model_path'] - cmd['lm_train_config'] = outputs['lm_model_config'] - cmd['batch_size'] = outputs['model_config']['batch_size'] - cmd['frontend_conf'] = frontend_conf - if frontend_conf is not None and 'fs' in frontend_conf: - cmd['fs']['model_fs'] = frontend_conf['fs'] - cmd['token_num_relax'] = token_num_relax - cmd['decoding_ind'] = decoding_ind - cmd['decoding_mode'] = decoding_mode - if outputs.__contains__('mvn_file'): - cmd['cmvn_file'] = outputs['mvn_file'] - model_config = self.model_cfg['model_config'] - if model_config.__contains__('vad_model') and self.vad_model != '': - self.vad_model = model_config['vad_model'] - if model_config.__contains__('vad_model_revision'): - self.vad_model_revision = model_config['vad_model_revision'] - if model_config.__contains__( - 'punc_model') and self.punc_model != '': - self.punc_model = model_config['punc_model'] - if model_config.__contains__('punc_model_revision'): - self.punc_model_revision = model_config['punc_model_revision'] - if model_config.__contains__( - 'timestamp_model') and self.timestamp_model != '': - self.timestamp_model = model_config['timestamp_model'] - if model_config.__contains__('timestamp_model_revision'): - self.timestamp_model_revision = model_config[ - 'timestamp_model_revision'] - update_local_model(model_config, model_path, extra_args) - self.load_vad_model(cmd) - self.load_punc_model(cmd) - self.load_lm_model(cmd) - self.load_timestamp_model(cmd) + cmd['beam_size'] = root['beam_size'] + cmd['penalty'] = root['penalty'] + cmd['maxlenratio'] = root['maxlenratio'] + cmd['minlenratio'] = root['minlenratio'] + cmd['ctc_weight'] = root['ctc_weight'] + cmd['lm_weight'] = root['lm_weight'] + cmd['asr_train_config'] = outputs['am_model_config'] + cmd['lm_file'] = outputs['lm_model_path'] + cmd['lm_train_config'] = outputs['lm_model_config'] + cmd['batch_size'] = outputs['model_config']['batch_size'] + cmd['frontend_conf'] = frontend_conf + if frontend_conf is not None and 'fs' in frontend_conf: + cmd['fs']['model_fs'] = frontend_conf['fs'] + cmd['token_num_relax'] = token_num_relax + cmd['decoding_ind'] = decoding_ind + cmd['decoding_mode'] = decoding_mode + if outputs.__contains__('mvn_file'): + cmd['cmvn_file'] = outputs['mvn_file'] + model_config = self.model_cfg['model_config'] + if model_config.__contains__('vad_model') and self.vad_model != '': + self.vad_model = model_config['vad_model'] + if model_config.__contains__('vad_model_revision'): + self.vad_model_revision = model_config['vad_model_revision'] + if model_config.__contains__('punc_model') and self.punc_model != '': + self.punc_model = model_config['punc_model'] + if model_config.__contains__('punc_model_revision'): + self.punc_model_revision = model_config['punc_model_revision'] + if model_config.__contains__( + 'timestamp_model') and self.timestamp_model != '': + self.timestamp_model = model_config['timestamp_model'] + if model_config.__contains__('timestamp_model_revision'): + self.timestamp_model_revision = model_config[ + 'timestamp_model_revision'] + update_local_model(model_config, model_path, extra_args) + self.load_vad_model(cmd) + self.load_punc_model(cmd) + self.load_lm_model(cmd) + self.load_timestamp_model(cmd) - user_args_dict = [ - 'output_dir', - 'batch_size', - 'mode', - 'ngpu', - 'beam_size', - 'ctc_weight', - 'lm_weight', - 'decoding_ind', - 'decoding_mode', - 'vad_model_file', - 'vad_infer_config', - 'vad_cmvn_file', - 'punc_model_file', - 'punc_infer_config', - 'param_dict', - ] + user_args_dict = [ + 'output_dir', + 'batch_size', + 'mode', + 'ngpu', + 'beam_size', + 'ctc_weight', + 'lm_weight', + 'decoding_ind', + 'decoding_mode', + 'vad_model_file', + 'vad_infer_config', + 'vad_cmvn_file', + 'punc_model_file', + 'punc_infer_config', + 'param_dict', + ] - for user_args in user_args_dict: - if user_args in extra_args and extra_args[ - user_args] is not None: - cmd[user_args] = extra_args[user_args] - - elif self.framework == Frameworks.tf: - cmd['fs']['model_fs'] = outputs['model_config']['fs'] - cmd['hop_length'] = outputs['model_config']['hop_length'] - cmd['feature_dims'] = outputs['model_config']['feature_dims'] - cmd['predictions_file'] = 'text' - cmd['cmvn_file'] = outputs['am_mvn_file'] - cmd['vocab_file'] = outputs['vocab_file'] - if 'idx_text' in outputs: - cmd['idx_text'] = outputs['idx_text'] - if 'sampled_ids' in outputs['model_config']: - cmd['sampled_ids'] = outputs['model_config']['sampled_ids'] - if 'sampled_lengths' in outputs['model_config']: - cmd['sampled_lengths'] = outputs['model_config'][ - 'sampled_lengths'] - else: - raise ValueError('model type is mismatching') + for user_args in user_args_dict: + if user_args in extra_args and extra_args[user_args] is not None: + cmd[user_args] = extra_args[user_args] + del extra_args[user_args] return cmd diff --git a/modelscope/pipelines/audio/lm_infer_pipeline.py b/modelscope/pipelines/audio/lm_infer_pipeline.py index 5e8928e5..75d835d6 100644 --- a/modelscope/pipelines/audio/lm_infer_pipeline.py +++ b/modelscope/pipelines/audio/lm_infer_pipeline.py @@ -80,7 +80,7 @@ class LanguageModelPipeline(Pipeline): mode=self.cmd['mode'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=ngpu, + ngpu=self.cmd['ngpu'], seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], @@ -194,6 +194,7 @@ class LanguageModelPipeline(Pipeline): for user_args in user_args_dict: if user_args in extra_args and extra_args[user_args] is not None: cmd[user_args] = extra_args[user_args] + del extra_args[user_args] return cmd diff --git a/modelscope/pipelines/audio/punctuation_processing_pipeline.py b/modelscope/pipelines/audio/punctuation_processing_pipeline.py index 54a26a8a..3ab3481d 100644 --- a/modelscope/pipelines/audio/punctuation_processing_pipeline.py +++ b/modelscope/pipelines/audio/punctuation_processing_pipeline.py @@ -54,7 +54,7 @@ class PunctuationProcessingPipeline(Pipeline): mode=self.cmd['mode'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=ngpu, + ngpu=self.cmd['ngpu'], seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], @@ -146,6 +146,7 @@ class PunctuationProcessingPipeline(Pipeline): for user_args in user_args_dict: if user_args in extra_args and extra_args[user_args] is not None: cmd[user_args] = extra_args[user_args] + del extra_args[user_args] return cmd diff --git a/modelscope/pipelines/audio/speaker_diarization_pipeline.py b/modelscope/pipelines/audio/speaker_diarization_pipeline.py index af6c81e8..71715ecd 100644 --- a/modelscope/pipelines/audio/speaker_diarization_pipeline.py +++ b/modelscope/pipelines/audio/speaker_diarization_pipeline.py @@ -77,7 +77,7 @@ class SpeakerDiarizationPipeline(Pipeline): output_dir=self.cmd['output_dir'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=ngpu, + ngpu=self.cmd['ngpu'], seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], @@ -205,6 +205,7 @@ class SpeakerDiarizationPipeline(Pipeline): cmd[user_args].update(extra_args[user_args]) else: cmd[user_args] = extra_args[user_args] + del extra_args[user_args] return cmd diff --git a/modelscope/pipelines/audio/speaker_verification_pipeline.py b/modelscope/pipelines/audio/speaker_verification_pipeline.py index 2f68cf95..e576885a 100644 --- a/modelscope/pipelines/audio/speaker_verification_pipeline.py +++ b/modelscope/pipelines/audio/speaker_verification_pipeline.py @@ -57,7 +57,7 @@ class SpeakerVerificationPipeline(Pipeline): output_dir=self.cmd['output_dir'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=ngpu, + ngpu=self.cmd['ngpu'], seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], @@ -172,6 +172,7 @@ class SpeakerVerificationPipeline(Pipeline): cmd[user_args].update(extra_args[user_args]) else: cmd[user_args] = extra_args[user_args] + del extra_args[user_args] return cmd diff --git a/modelscope/pipelines/audio/timestamp_pipeline.py b/modelscope/pipelines/audio/timestamp_pipeline.py index 13fb0f82..0968b359 100644 --- a/modelscope/pipelines/audio/timestamp_pipeline.py +++ b/modelscope/pipelines/audio/timestamp_pipeline.py @@ -75,7 +75,7 @@ class TimestampPipeline(Pipeline): mode=self.cmd['mode'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=ngpu, + ngpu=self.cmd['ngpu'], seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], @@ -269,6 +269,7 @@ class TimestampPipeline(Pipeline): for user_args in user_args_dict: if user_args in extra_args and extra_args[user_args] is not None: cmd[user_args] = extra_args[user_args] + del extra_args[user_args] return cmd diff --git a/modelscope/pipelines/audio/voice_activity_detection_pipeline.py b/modelscope/pipelines/audio/voice_activity_detection_pipeline.py index f1bb86b6..0121b242 100644 --- a/modelscope/pipelines/audio/voice_activity_detection_pipeline.py +++ b/modelscope/pipelines/audio/voice_activity_detection_pipeline.py @@ -56,7 +56,7 @@ class VoiceActivityDetectionPipeline(Pipeline): mode=self.cmd['mode'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=ngpu, + ngpu=self.cmd['ngpu'], seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], @@ -214,6 +214,7 @@ class VoiceActivityDetectionPipeline(Pipeline): for user_args in user_args_dict: if user_args in extra_args and extra_args[user_args] is not None: cmd[user_args] = extra_args[user_args] + del extra_args[user_args] return cmd From 5804ad2dc1333672056424d111ec0c6723988792 Mon Sep 17 00:00:00 2001 From: "hemu.zp" Date: Tue, 16 May 2023 14:31:26 +0800 Subject: [PATCH 42/76] update multi_modal_embedding example Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12626062 --- .../finetune_multi_modal_embedding.py | 49 ++++++++++--------- .../multi_modal_embedding/run_train.sh | 5 +- .../finetune_text_classification.py | 3 +- .../finetune_text_generation.py | 4 +- .../trainers/multi_modal/clip/clip_trainer.py | 5 +- 5 files changed, 37 insertions(+), 29 deletions(-) diff --git a/examples/pytorch/multi_modal_embedding/finetune_multi_modal_embedding.py b/examples/pytorch/multi_modal_embedding/finetune_multi_modal_embedding.py index b3c325cf..7b4cfbb8 100644 --- a/examples/pytorch/multi_modal_embedding/finetune_multi_modal_embedding.py +++ b/examples/pytorch/multi_modal_embedding/finetune_multi_modal_embedding.py @@ -1,15 +1,13 @@ import os from dataclasses import dataclass, field -from functools import partial +from modelscope import MsDataset, TrainingArgs from modelscope.metainfo import Trainers -from modelscope.msdatasets import MsDataset from modelscope.trainers import build_trainer -from modelscope.trainers.args import (TrainingArgs, get_flatten_value, - set_flatten_value) +from modelscope.trainers.training_args import set_flatten_value -@dataclass +@dataclass(init=False) class MultiModalEmbeddingArguments(TrainingArgs): trainer: str = field( @@ -17,6 +15,12 @@ class MultiModalEmbeddingArguments(TrainingArgs): 'help': 'The trainer used', }) + work_dir: str = field( + default='./tmp', + metadata={ + 'help': 'The working path for saving checkpoint', + }) + use_fp16: bool = field( default=None, metadata={ @@ -35,7 +39,6 @@ class MultiModalEmbeddingArguments(TrainingArgs): default=None, metadata={ 'cfg_node': 'train.optimizer_hparams', - 'cfg_getter': partial(get_flatten_value, exclusions=['lr']), 'cfg_setter': set_flatten_value, 'help': 'The optimizer init params except `lr`', }) @@ -51,7 +54,6 @@ class MultiModalEmbeddingArguments(TrainingArgs): default=None, metadata={ 'cfg_node': 'dataset.column_map', - 'cfg_getter': get_flatten_value, 'cfg_setter': set_flatten_value, 'help': 'The column map for dataset', }) @@ -67,7 +69,6 @@ class MultiModalEmbeddingArguments(TrainingArgs): default=None, metadata={ 'cfg_node': 'train.lr_scheduler_hook', - 'cfg_getter': get_flatten_value, 'cfg_setter': set_flatten_value, 'help': 'The parameters for lr scheduler hook', }) @@ -76,7 +77,6 @@ class MultiModalEmbeddingArguments(TrainingArgs): default=None, metadata={ 'cfg_node': 'train.optimizer_hook', - 'cfg_getter': get_flatten_value, 'cfg_setter': set_flatten_value, 'help': 'The parameters for optimizer hook', }) @@ -92,23 +92,28 @@ class MultiModalEmbeddingArguments(TrainingArgs): 'help': 'The data parallel world size', }) - def __call__(self, config): - config = super().__call__(config) - config.merge_from_dict({'pretrained_model.model_name': self.model}) - if self.clip_clamp: - config.train.hooks.append({'type': 'ClipClampLogitScaleHook'}) - if self.world_size > 1: - config.train.launcher = 'pytorch' - return config + +config, args = MultiModalEmbeddingArguments().parse_cli().to_config() +print(config, args) -args = MultiModalEmbeddingArguments.from_cli(task='multi-modal-embedding') -print(args) +def cfg_modify_fn(cfg): + if args.use_model_config: + cfg.merge_from_dict(config) + else: + cfg = config + cfg.merge_from_dict({'pretrained_model.model_name': args.model}) + if args.clip_clamp: + cfg.train.hooks.append({'type': 'ClipClampLogitScaleHook'}) + if args.world_size > 1: + cfg.train.launcher = 'pytorch' + return cfg + train_dataset = MsDataset.load( - args.dataset_name, namespace='modelscope', split='train') + args.train_dataset_name, namespace='modelscope', split='train') eval_dataset = MsDataset.load( - args.dataset_name, namespace='modelscope', split='validation') + args.train_dataset_name, namespace='modelscope', split='validation') os.makedirs(args.work_dir, exist_ok=True) kwargs = dict( @@ -116,6 +121,6 @@ kwargs = dict( train_dataset=train_dataset, eval_dataset=eval_dataset, work_dir=args.work_dir, - cfg_modify_fn=args) + cfg_modify_fn=cfg_modify_fn) trainer = build_trainer(name=args.trainer, default_args=kwargs) trainer.train() diff --git a/examples/pytorch/multi_modal_embedding/run_train.sh b/examples/pytorch/multi_modal_embedding/run_train.sh index 89eef73e..3974405b 100644 --- a/examples/pytorch/multi_modal_embedding/run_train.sh +++ b/examples/pytorch/multi_modal_embedding/run_train.sh @@ -6,14 +6,16 @@ PYTHONPATH=. torchrun --nproc_per_node $DATA_PARALLEL_SIZE \ --trainer 'clip-multi-modal-embedding' \ --work_dir './workspace/ckpts/clip' \ --model 'damo/multi-modal_clip-vit-base-patch16_zh' \ - --dataset_name 'muge' \ + --train_dataset_name 'muge' \ --dataset_column_map 'img=image,text=query' \ --max_epochs 1 \ --use_fp16 true \ --per_device_train_batch_size 180 \ + --train_data_worker 0 \ --train_shuffle true \ --train_drop_last true \ --per_device_eval_batch_size 128 \ + --eval_data_worker 0 \ --eval_shuffle true \ --eval_drop_last true \ --save_ckpt_best true \ @@ -33,3 +35,4 @@ PYTHONPATH=. torchrun --nproc_per_node $DATA_PARALLEL_SIZE \ --optimizer_hook 'type=TorchAMPOptimizerHook,cumulative_iters=1,loss_keys=loss' \ --clip_clamp true \ --world_size $DATA_PARALLEL_SIZE \ + --use_model_config true \ diff --git a/examples/pytorch/text_classification/finetune_text_classification.py b/examples/pytorch/text_classification/finetune_text_classification.py index 73d15783..dfcb7b4d 100644 --- a/examples/pytorch/text_classification/finetune_text_classification.py +++ b/examples/pytorch/text_classification/finetune_text_classification.py @@ -2,7 +2,8 @@ import os from dataclasses import dataclass, field from modelscope import (EpochBasedTrainer, MsDataset, TrainingArgs, - build_dataset_from_file, build_trainer) + build_dataset_from_file) +from modelscope.trainers import build_trainer def set_labels(labels): diff --git a/examples/pytorch/text_generation/finetune_text_generation.py b/examples/pytorch/text_generation/finetune_text_generation.py index f9e47bf3..a89970e8 100644 --- a/examples/pytorch/text_generation/finetune_text_generation.py +++ b/examples/pytorch/text_generation/finetune_text_generation.py @@ -1,8 +1,8 @@ from dataclasses import dataclass, field -from modelscope import (EpochBasedTrainer, MsDataset, TrainingArgs, - build_trainer) +from modelscope import EpochBasedTrainer, MsDataset, TrainingArgs from modelscope.metainfo import Trainers +from modelscope.trainers import build_trainer @dataclass(init=False) diff --git a/modelscope/trainers/multi_modal/clip/clip_trainer.py b/modelscope/trainers/multi_modal/clip/clip_trainer.py index b0415bc2..ae00232f 100644 --- a/modelscope/trainers/multi_modal/clip/clip_trainer.py +++ b/modelscope/trainers/multi_modal/clip/clip_trainer.py @@ -176,11 +176,10 @@ class CLIPTrainer(EpochBasedTrainer): self.dataset_cfg = cfg.dataset if hasattr(self.dataset_cfg, 'column_map'): # cases where dataset key names are not "img" and "text" - img_key_name = getattr(self.dataset_cfg.column_map, 'img', 'img') + img_key_name = self.dataset_cfg['column_map'].get('img', 'img') preprocessor[ConfigKeys.train].set_input_img_key(img_key_name) preprocessor[ConfigKeys.val].set_input_img_key(img_key_name) - text_key_name = getattr(self.dataset_cfg.column_map, 'text', - 'text') + text_key_name = self.dataset_cfg['column_map'].get('text', 'text') preprocessor[ConfigKeys.train].set_input_text_key(text_key_name) preprocessor[ConfigKeys.val].set_input_text_key(text_key_name) self.global_batch_size = cfg.train.dataloader.batch_size_per_gpu * world_size From 71a80173e5e7e882bf9b3c70188d20eca94c67eb Mon Sep 17 00:00:00 2001 From: zsl01670416 Date: Tue, 16 May 2023 14:32:43 +0800 Subject: [PATCH 43/76] modify examples text_classification, text_generation, token_classification for improved trainer 1.text_classification/run_train.sh 2.text_generation/run_train_mt5.sh, run_train_palm.sh 3.token_classification/finetune_token_classification.py, run_train_mgeo.sh, run_train_structbert.sh above files were modified to adapt to improved trainer Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12629010 --- .../pytorch/text_classification/run_train.sh | 5 +- .../pytorch/text_generation/run_train_mt5.sh | 13 ++ .../pytorch/text_generation/run_train_palm.sh | 5 +- .../finetune_token_classification.py | 137 ++++++++++++------ .../token_classification/run_train_mgeo.sh | 21 ++- .../run_train_structbert.sh | 22 ++- 6 files changed, 137 insertions(+), 66 deletions(-) create mode 100644 examples/pytorch/text_generation/run_train_mt5.sh diff --git a/examples/pytorch/text_classification/run_train.sh b/examples/pytorch/text_classification/run_train.sh index f46fb72a..e91a9996 100644 --- a/examples/pytorch/text_classification/run_train.sh +++ b/examples/pytorch/text_classification/run_train.sh @@ -1,4 +1,5 @@ PYTHONPATH=. python examples/pytorch/text_classification/finetune_text_classification.py \ + --task 'text-classification' \ --model 'damo/nlp_structbert_backbone_base_std' \ --train_dataset_name 'clue' \ --train_subset_name 'tnews' \ @@ -7,7 +8,9 @@ PYTHONPATH=. python examples/pytorch/text_classification/finetune_text_classific --model.num_labels 15 \ --labels '0,1,2,3,4,5,6,7,8,9,10,11,12,13,14' \ --preprocessor 'sen-cls-tokenizer' \ + --use_model_config True \ + --max_epochs 1 \ --train.dataloader.workers_per_gpu 0 \ --evaluation.dataloader.workers_per_gpu 0 \ --train.optimizer.lr 1e-5 \ - --use_model_config true \ + --eval_metrics 'seq-cls-metric' \ diff --git a/examples/pytorch/text_generation/run_train_mt5.sh b/examples/pytorch/text_generation/run_train_mt5.sh new file mode 100644 index 00000000..6d032d6e --- /dev/null +++ b/examples/pytorch/text_generation/run_train_mt5.sh @@ -0,0 +1,13 @@ +PYTHONPATH=. torchrun examples/pytorch/text_generation/finetune_text_generation.py \ + --trainer 'text-generation-trainer' \ + --work_dir './tmp' \ + --task 'text2text-generation' \ + --model 'damo/nlp_mt5_zero-shot-augment_chinese-base' \ + --train_dataset_name 'DuReader_robust-QG' \ + --src_txt 'text1' \ + --tgt_txt 'text2' \ + --max_epochs 1 \ + --use_model_config True \ + --per_device_train_batch_size 8 \ + --lr 1e-3 \ + --lr_scheduler 'noam' \ diff --git a/examples/pytorch/text_generation/run_train_palm.sh b/examples/pytorch/text_generation/run_train_palm.sh index ff88ce7d..68b9e89d 100644 --- a/examples/pytorch/text_generation/run_train_palm.sh +++ b/examples/pytorch/text_generation/run_train_palm.sh @@ -2,10 +2,11 @@ PYTHONPATH=. torchrun examples/pytorch/text_generation/finetune_text_generation. --trainer 'text-generation-trainer' \ --work_dir './tmp' \ --model 'damo/nlp_palm2.0_pretrained_chinese-base' \ - --dataset_name 'DuReader_robust-QG' \ + --train_dataset_name 'DuReader_robust-QG' \ --src_txt 'text1' \ --tgt_txt 'text2' \ - --max_epochs 15 \ + --max_epochs 1 \ + --use_model_config True \ --per_device_train_batch_size 8 \ --lr 1e-3 \ --lr_scheduler 'noam' \ diff --git a/examples/pytorch/token_classification/finetune_token_classification.py b/examples/pytorch/token_classification/finetune_token_classification.py index e06f6f48..3f9de791 100644 --- a/examples/pytorch/token_classification/finetune_token_classification.py +++ b/examples/pytorch/token_classification/finetune_token_classification.py @@ -1,20 +1,22 @@ from dataclasses import dataclass, field -from modelscope.metainfo import Trainers -from modelscope.msdatasets import MsDataset -from modelscope.trainers import build_trainer -from modelscope.trainers.args import (TrainingArgs, get_flatten_value, - set_flatten_value) +from modelscope import (EpochBasedTrainer, MsDataset, TrainingArgs, + build_dataset_from_file) -@dataclass +@dataclass(init=False) class TokenClassificationArguments(TrainingArgs): - trainer: str = field( - default=Trainers.default, metadata={ + default=None, metadata={ 'help': 'The trainer used', }) + work_dir: str = field( + default='./tmp', + metadata={ + 'help': 'The working path for saving checkpoint', + }) + preprocessor: str = field( default=None, metadata={ @@ -29,60 +31,99 @@ class TokenClassificationArguments(TrainingArgs): 'cfg_node': 'preprocessor.padding' }) - train_dataset_params: str = field( + mode: str = field( + default='inference', + metadata={ + 'help': 'The preprocessor padding', + 'cfg_node': 'preprocessor.mode' + }) + + first_sequence: str = field( default=None, metadata={ - 'cfg_node': 'dataset.train', - 'cfg_getter': get_flatten_value, - 'cfg_setter': set_flatten_value, + 'cfg_node': 'preprocessor.first_sequence', 'help': 'The parameters for train dataset', }) - def __call__(self, config): - config = super().__call__(config) - if config.safe_get('dataset.train.label') == 'ner_tags': - ner_tags_labels = train_dataset['ner_tags'] + eval_dataset[ - 'ner_tags'] - label_enumerate_values = self._get_label_list(ner_tags_labels) - config.merge_from_dict( - {'dataset.train.labels': label_enumerate_values}) - if config.train.lr_scheduler.type == 'LinearLR': - config.train.lr_scheduler['total_iters'] = \ - int(len(train_dataset) / self.per_device_train_batch_size) * self.max_epochs - return config + label: str = field( + default=None, + metadata={ + 'cfg_node': 'preprocessor.label', + 'help': 'The parameters for train dataset', + }) - # TODO: Future performance optimization in MsDataset - @staticmethod - def _get_label_list(labels): - unique_labels = set() - for label in labels: - unique_labels = unique_labels | set(label) - label_list = list(unique_labels) - label_list.sort() - return label_list + sequence_length: int = field( + default=128, + metadata={ + 'cfg_node': 'preprocessor.sequence_length', + 'help': 'The parameters for train dataset', + }) -args = TokenClassificationArguments.from_cli(task='token-classification') +training_args = TokenClassificationArguments().parse_cli() +config, args = training_args.to_config() print(args) -# load dataset -train_dataset = MsDataset.load( - args.dataset_name, - subset_name=args.subset_name, - split='train', - namespace='damo')['train'] -eval_dataset = MsDataset.load( - args.dataset_name, - subset_name=args.subset_name, - split='validation', - namespace='damo')['validation'] + +def get_label_list(labels): + unique_labels = set() + for label in labels: + unique_labels = unique_labels | set(label) + label_list = list(unique_labels) + label_list.sort() + return label_list + + +def cfg_modify_fn(cfg): + if args.use_model_config: + cfg.merge_from_dict(config) + else: + cfg = config + labels = train_dataset[training_args.label] + validation_dataset[ + training_args.label] + label_enumerate_values = get_label_list(labels) + cfg.merge_from_dict({ + 'preprocessor.label2id': + {label: id + for id, label in enumerate(label_enumerate_values)} + }) + cfg.merge_from_dict({'model.num_labels': len(label_enumerate_values)}) + cfg.merge_from_dict({'preprocessor.use_fast': True}) + cfg.merge_from_dict({ + 'evaluation.metrics': { + 'type': 'token-cls-metric', + 'label2id': + {label: id + for id, label in enumerate(label_enumerate_values)} + } + }) + if cfg.train.lr_scheduler.type == 'LinearLR': + cfg.train.lr_scheduler['total_iters'] = \ + int(len(train_dataset) / cfg.train.dataloader.batch_size_per_gpu) * cfg.train.max_epochs + return cfg + + +if args.dataset_json_file is None: + train_dataset = MsDataset.load( + args.train_dataset_name, + subset_name=args.train_subset_name, + split='train', + namespace=args.train_dataset_namespace)['train'] + validation_dataset = MsDataset.load( + args.train_dataset_name, + subset_name=args.train_subset_name, + split='validation', + namespace=args.train_dataset_namespace)['validation'] +else: + train_dataset, validation_dataset = build_dataset_from_file( + args.dataset_json_file) kwargs = dict( model=args.model, train_dataset=train_dataset, - eval_dataset=eval_dataset, + eval_dataset=validation_dataset, work_dir=args.work_dir, - cfg_modify_fn=args) + cfg_modify_fn=cfg_modify_fn) -trainer = build_trainer(name=args.trainer, default_args=kwargs) +trainer = EpochBasedTrainer(**kwargs) trainer.train() diff --git a/examples/pytorch/token_classification/run_train_mgeo.sh b/examples/pytorch/token_classification/run_train_mgeo.sh index f80af84f..1e384ec5 100644 --- a/examples/pytorch/token_classification/run_train_mgeo.sh +++ b/examples/pytorch/token_classification/run_train_mgeo.sh @@ -1,15 +1,22 @@ -PYTHONPATH=. torchrun examples/pytorch/token_classification/finetune_token_classification.py \ +PYTHONPATH=. python examples/pytorch/token_classification/finetune_token_classification.py \ + --task 'token-classification' \ --trainer 'nlp-base-trainer' \ --work_dir './tmp' \ --model 'damo/mgeo_backbone_chinese_base' \ - --dataset_name 'GeoGLUE' \ - --subset_name 'GeoETA' \ - --train_dataset_params 'first_sequence=tokens,label=ner_tags,sequence_length=128' \ + --train_dataset_name 'GeoGLUE' \ + --train_subset_name 'GeoETA' \ + --train_dataset_namespace 'damo' \ + --first_sequence 'tokens' \ + --eval_strategy by_step \ + --eval_interval 10 \ + --label 'ner_tags' \ + --sequence_length 128 \ --preprocessor 'token-cls-tokenizer' \ --preprocessor_padding 'max_length' \ --max_epochs 1 \ + --mode 'inference' \ + --use_model_config True \ --per_device_train_batch_size 32 \ + --train_data_worker 0 \ + --eval_data_worker 0 \ --lr 3e-5 \ - --save_ckpt_strategy 'by_epoch' \ - --logging_interval 100 \ - --eval_strategy 'by_epoch' \ diff --git a/examples/pytorch/token_classification/run_train_structbert.sh b/examples/pytorch/token_classification/run_train_structbert.sh index 28967f60..a44c4519 100644 --- a/examples/pytorch/token_classification/run_train_structbert.sh +++ b/examples/pytorch/token_classification/run_train_structbert.sh @@ -1,16 +1,22 @@ -PYTHONPATH=. torchrun examples/pytorch/token_classification/finetune_token_classification.py \ +PYTHONPATH=. python examples/pytorch/token_classification/finetune_token_classification.py \ + --task 'token-classification' \ --trainer 'nlp-base-trainer' \ --work_dir './tmp' \ --model 'damo/nlp_structbert_backbone_base_std' \ - --dataset_name 'GeoGLUE' \ - --subset_name 'GeoETA' \ - --train_dataset_params 'first_sequence=tokens,label=ner_tags,sequence_length=128' \ + --train_dataset_name 'GeoGLUE' \ + --train_subset_name 'GeoETA' \ + --train_dataset_namespace 'damo' \ + --first_sequence 'tokens' \ + --eval_strategy by_step \ + --eval_interval 20 \ + --label 'ner_tags' \ + --sequence_length 128 \ --preprocessor 'token-cls-tokenizer' \ --preprocessor_padding 'max_length' \ --max_epochs 2 \ + --mode 'inference' \ + --use_model_config True \ --per_device_train_batch_size 32 \ + --train_data_worker 0 \ + --eval_data_worker 0 \ --lr 3e-5 \ - --save_ckpt_strategy 'by_epoch' \ - --logging_interval 1 \ - --eval_strategy 'by_step' \ - --eval_interval 20 \ From 38fa4cceed0ae2ba8ed0974e33fe63af167635a5 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 16 May 2023 15:27:17 +0800 Subject: [PATCH 44/76] fix transformer example and fix some bugs Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12626375 --- .../finetune_image_classification.py | 42 +++++++++------- .../pytorch/image_classification/run_train.sh | 5 +- .../finetune_stable_diffusion.py | 34 ++++++------- .../pytorch/stable_diffusion/run_train.sh | 7 +-- .../pytorch/transformers/configuration.json | 1 - .../finetune_transformers_model.py | 48 +++++++++++++------ examples/pytorch/transformers/run_train.sh | 13 ++++- .../hooks/checkpoint/checkpoint_hook.py | 2 +- modelscope/trainers/training_args.py | 22 +++++++-- 9 files changed, 115 insertions(+), 59 deletions(-) delete mode 100644 examples/pytorch/transformers/configuration.json diff --git a/examples/pytorch/image_classification/finetune_image_classification.py b/examples/pytorch/image_classification/finetune_image_classification.py index 1e7fceb1..e5bb9bdd 100644 --- a/examples/pytorch/image_classification/finetune_image_classification.py +++ b/examples/pytorch/image_classification/finetune_image_classification.py @@ -1,13 +1,12 @@ import os from dataclasses import dataclass, field +from modelscope import MsDataset, TrainingArgs from modelscope.metainfo import Trainers -from modelscope.msdatasets.ms_dataset import MsDataset -from modelscope.trainers.args import TrainingArgs from modelscope.trainers.builder import build_trainer -@dataclass +@dataclass(init=False) class ImageClassificationTrainingArgs(TrainingArgs): num_classes: int = field( default=None, @@ -46,26 +45,35 @@ def create_dataset(name, split): dataset_name, namespace=namespace, subset_name='default', split=split) -def train(): - args = ImageClassificationTrainingArgs.from_cli( - model='damo/cv_vit-base_image-classification_ImageNet-labels', - max_epochs=1, - lr=1e-4, - optimizer='AdamW', - warmup_iters=1, - topk=(1, )) - if args.dataset_name is not None: - train_dataset = create_dataset(args.dataset_name, split='train') - val_dataset = create_dataset(args.dataset_name, split='validation') +training_args = ImageClassificationTrainingArgs( + model='damo/cv_vit-base_image-classification_ImageNet-labels', + max_epochs=1, + lr=1e-4, + optimizer='AdamW', + warmup_iters=1, + topk=(1, )).parse_cli() +config, args = training_args.to_config() + + +def cfg_modify_fn(cfg): + if args.use_model_config: + cfg.merge_from_dict(config) else: - train_dataset = create_dataset(args.train_dataset_name, split='train') - val_dataset = create_dataset(args.val_dataset_name, split='validation') + cfg = config + return cfg + + +def train(): + train_dataset = create_dataset( + training_args.train_dataset_name, split=training_args.train_split) + val_dataset = create_dataset( + training_args.val_dataset_name, split=training_args.val_split) kwargs = dict( model=args.model, # model id train_dataset=train_dataset, # training dataset eval_dataset=val_dataset, # validation dataset - cfg_modify_fn=args # callback to modify configuration + cfg_modify_fn=cfg_modify_fn # callback to modify configuration ) # in distributed training, specify pytorch launcher diff --git a/examples/pytorch/image_classification/run_train.sh b/examples/pytorch/image_classification/run_train.sh index 5a7b3a09..ad560424 100644 --- a/examples/pytorch/image_classification/run_train.sh +++ b/examples/pytorch/image_classification/run_train.sh @@ -2,4 +2,7 @@ PYTHONPATH=. python -m torch.distributed.launch --nproc_per_node=2 \ examples/pytorch/image_classification/finetune_image_classification.py \ --num_classes 2 \ --train_dataset_name 'tany0699/cats_and_dogs' \ - --val_dataset_name 'tany0699/cats_and_dogs' + --val_dataset_name 'tany0699/cats_and_dogs' \ + --train_split train \ + --val_split validation \ + --use_model_config true \ diff --git a/examples/pytorch/stable_diffusion/finetune_stable_diffusion.py b/examples/pytorch/stable_diffusion/finetune_stable_diffusion.py index bd05097d..28ba853c 100644 --- a/examples/pytorch/stable_diffusion/finetune_stable_diffusion.py +++ b/examples/pytorch/stable_diffusion/finetune_stable_diffusion.py @@ -4,30 +4,32 @@ from modelscope.msdatasets import MsDataset from modelscope.trainers import EpochBasedTrainer, build_trainer from modelscope.trainers.training_args import TrainingArgs - -@dataclass -class StableDiffusionArguments(TrainingArgs): - - def __call__(self, config): - config = super().__call__(config) - config.train.lr_scheduler.T_max = self.max_epochs - config.model.inference = False - return config - - -args = StableDiffusionArguments.from_cli(task='efficient-diffusion-tuning') +training_args = TrainingArgs(task='efficient-diffusion-tuning').parse_cli() +config, args = training_args.to_config() print(args) -dataset = MsDataset.load(args.dataset_name, namespace=args.namespace) +dataset = MsDataset.load( + args.train_dataset_name, namespace=args.train_dataset_namespace) train_dataset = dataset['train'] validation_dataset = dataset['validation'] + +def cfg_modify_fn(cfg): + if args.use_model_config: + cfg.merge_from_dict(config) + else: + cfg = config + cfg.train.lr_scheduler.T_max = training_args.max_epochs + cfg.model.inference = False + return cfg + + kwargs = dict( - model=args.model, - work_dir=args.work_dir, + model=training_args.model, + work_dir=training_args.work_dir, train_dataset=train_dataset, eval_dataset=validation_dataset, - cfg_modify_fn=args) + cfg_modify_fn=cfg_modify_fn) trainer: EpochBasedTrainer = build_trainer(name='trainer', default_args=kwargs) trainer.train() diff --git a/examples/pytorch/stable_diffusion/run_train.sh b/examples/pytorch/stable_diffusion/run_train.sh index e3b78691..0e551942 100644 --- a/examples/pytorch/stable_diffusion/run_train.sh +++ b/examples/pytorch/stable_diffusion/run_train.sh @@ -1,11 +1,12 @@ PYTHONPATH=. torchrun examples/pytorch/stable_diffusion/finetune_stable_diffusion.py \ --model 'damo/multi-modal_efficient-diffusion-tuning-lora' \ --work_dir './tmp/stable_diffusion_tuning' \ - --namespace 'damo' \ - --dataset_name 'controlnet_dataset_condition_fill50k' \ + --train_dataset_namespace 'damo' \ + --train_dataset_name 'controlnet_dataset_condition_fill50k' \ --max_epochs 1 \ --save_ckpt_strategy 'by_epoch' \ --logging_interval 100 \ --train.dataloader.workers_per_gpu 0 \ --evaluation.dataloader.workers_per_gpu 0 \ - --train.optimizer.lr 1e-5 + --train.optimizer.lr 1e-5 \ + --use_model_config true diff --git a/examples/pytorch/transformers/configuration.json b/examples/pytorch/transformers/configuration.json deleted file mode 100644 index df6a73c8..00000000 --- a/examples/pytorch/transformers/configuration.json +++ /dev/null @@ -1 +0,0 @@ -{"framework":"pytorch","train":{"work_dir":"/tmp","max_epochs":10,"dataloader":{"batch_size_per_gpu":16,"workers_per_gpu":0},"optimizer":{"type":"SGD","lr":0.001},"lr_scheduler":{"type":"StepLR","step_size":2},"hooks":[{"type":"CheckpointHook","interval":1}]},"evaluation":{"dataloader":{"batch_size_per_gpu":16,"workers_per_gpu":0,"shuffle":false}}} diff --git a/examples/pytorch/transformers/finetune_transformers_model.py b/examples/pytorch/transformers/finetune_transformers_model.py index bbfb807a..5110f751 100644 --- a/examples/pytorch/transformers/finetune_transformers_model.py +++ b/examples/pytorch/transformers/finetune_transformers_model.py @@ -5,11 +5,11 @@ from datasets import load_dataset from transformers import (BertForSequenceClassification, BertTokenizerFast, default_data_collator) +from modelscope import TrainingArgs from modelscope.trainers import EpochBasedTrainer, build_trainer -from modelscope.trainers.default_config import DEFAULT_CONFIG, TrainingArgs -@dataclass +@dataclass(init=False) class TransformersArguments(TrainingArgs): num_labels: int = field( @@ -17,13 +17,27 @@ class TransformersArguments(TrainingArgs): 'help': 'The number of labels', }) + sentence: str = field( + default=None, metadata={ + 'help': 'The sentence key', + }) -args = TransformersArguments.from_cli( - task='text-classification', eval_metrics='seq-cls-metric') + label: str = field( + default=None, metadata={ + 'help': 'The label key', + }) -print(args) -dataset = load_dataset(args.dataset_name, args.subset_name) +training_args = TransformersArguments( + task='text-classification', eval_metrics='seq-cls-metric').parse_cli() +config, args = training_args.to_config() + +print(config, args) + +train_dataset = load_dataset( + args.train_dataset_name, args.train_subset_name, split=args.train_split) +val_dataset = load_dataset( + args.val_dataset_name, args.val_subset_name, split=args.val_split) model = BertForSequenceClassification.from_pretrained( args.model, num_labels=args.num_labels) @@ -31,26 +45,30 @@ tokenizer = BertTokenizerFast.from_pretrained(args.model) def tokenize_sentence(row): - return tokenizer(row['sentence'], padding='max_length', max_length=128) + return tokenizer( + row[training_args.sentence], padding='max_length', max_length=128) # Extra columns, Rename columns -dataset = dataset.map(tokenize_sentence).remove_columns(['sentence', - 'idx']).rename_column( - 'label', 'labels') +train_dataset = train_dataset.map(tokenize_sentence) +val_dataset = val_dataset.map(tokenize_sentence) +if training_args.label != 'labels': + train_dataset = train_dataset.rename_columns( + {training_args.label: 'labels'}) + val_dataset = val_dataset.rename_columns({training_args.label: 'labels'}) cfg_file = os.path.join(args.work_dir or './', 'configuration.json') -DEFAULT_CONFIG.dump(cfg_file) +config.dump(cfg_file) kwargs = dict( model=model, cfg_file=cfg_file, # data_collator data_collator=default_data_collator, - train_dataset=dataset['train'], - eval_dataset=dataset['validation'], - seed=args.seed, - cfg_modify_fn=args) + train_dataset=train_dataset, + eval_dataset=val_dataset, + remove_unused_data=True, + seed=args.seed) os.environ['LOCAL_RANK'] = str(args.local_rank) trainer: EpochBasedTrainer = build_trainer(name='trainer', default_args=kwargs) diff --git a/examples/pytorch/transformers/run_train.sh b/examples/pytorch/transformers/run_train.sh index c76c4636..94e5ef75 100644 --- a/examples/pytorch/transformers/run_train.sh +++ b/examples/pytorch/transformers/run_train.sh @@ -1,5 +1,14 @@ PYTHONPATH=. python examples/pytorch/transformers/finetune_transformers_model.py \ --model bert-base-uncased \ --num_labels 15 \ - --dataset_name clue \ - --subset_name tnews + --train_dataset_name clue \ + --train_subset_name tnews \ + --train_split train \ + --val_dataset_name clue \ + --val_subset_name tnews \ + --train_split train \ + --val_split validation \ + --sentence sentence \ + --label label \ + --eval_strategy by_step \ + --eval_interval 100 diff --git a/modelscope/trainers/hooks/checkpoint/checkpoint_hook.py b/modelscope/trainers/hooks/checkpoint/checkpoint_hook.py index e531a325..4b14a13f 100644 --- a/modelscope/trainers/hooks/checkpoint/checkpoint_hook.py +++ b/modelscope/trainers/hooks/checkpoint/checkpoint_hook.py @@ -348,7 +348,7 @@ class BestCkptSaverHook(CheckpointHook): self._do_save(trainer, 'by_step') def _should_save(self, trainer): - return self._is_best_metric(trainer.metric_values) and self.save_best + return self.save_best and self._is_best_metric(trainer.metric_values) def _is_best_metric(self, metric_values): if metric_values is None: diff --git a/modelscope/trainers/training_args.py b/modelscope/trainers/training_args.py index 260f2eb9..b7236163 100644 --- a/modelscope/trainers/training_args.py +++ b/modelscope/trainers/training_args.py @@ -51,6 +51,16 @@ class DatasetArgs: 'help': 'The subset name used for evaluating, can be None', }) + train_split: str = field( + default=None, metadata={ + 'help': 'The split of train dataset', + }) + + val_split: str = field( + default=None, metadata={ + 'help': 'The split of val dataset', + }) + train_dataset_namespace: str = field( default=None, metadata={ @@ -431,7 +441,10 @@ class TrainingArgs(DatasetArgs, TrainArgs, ModelArgs): """ parser = CliArgumentParser(self) args, unknown = parser.parse_known_args(parser_args) - unknown = [item for item in unknown if item not in ('\\', '\n')] + unknown = [ + item for item in unknown + if item not in ('\\', '\n') and '--local-rank=' not in item + ] _unknown = {} for i in range(0, len(unknown), 2): _unknown[unknown[i].replace('-', '')] = parse_value(unknown[i + 1]) @@ -460,8 +473,11 @@ class TrainingArgs(DatasetArgs, TrainArgs, ModelArgs): cfg_setter = f.metadata.get('cfg_setter') or (lambda x: x) if cfg_node is not None: if f.name in self.manual_args or not ignore_default_config: - cfg.merge_from_dict( - {cfg_node: cfg_setter(getattr(self, f.name))}) + if isinstance(cfg_node, str): + cfg_node = [cfg_node] + for _node in cfg_node: + cfg.merge_from_dict( + {_node: cfg_setter(getattr(self, f.name))}) else: args_dict[f.name] = getattr(self, f.name) From c6ef9024d75c7510d66fb63eb0c9260627568c7a Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 16 May 2023 21:37:12 +0800 Subject: [PATCH 45/76] Fix distributed hook to lazyimport; fix an import bug Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12638653 --- modelscope/trainers/hooks/__init__.py | 10 +++++++--- modelscope/trainers/hooks/distributed/__init__.py | 3 --- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/modelscope/trainers/hooks/__init__.py b/modelscope/trainers/hooks/__init__.py index 5485f454..072105be 100644 --- a/modelscope/trainers/hooks/__init__.py +++ b/modelscope/trainers/hooks/__init__.py @@ -16,7 +16,9 @@ if TYPE_CHECKING: OptimizerHook, TorchAMPOptimizerHook) from .priority import Priority, get_priority from .checkpoint import CheckpointHook, LoadCheckpointHook, BestCkptSaverHook - from .distributed import DDPHook, DeepspeedHook, MegatronHook + from .distributed.ddp_hook import DDPHook + from .distributed.deepspeed_hook import DeepspeedHook + from .distributed.megatron_hook import MegatronHook else: _import_structure = { @@ -35,8 +37,10 @@ else: ], 'checkpoint': ['CheckpointHook', 'LoadCheckpointHook', 'BestCkptSaverHook'], - 'distributed': ['DDPHook', 'DeepspeedHook', 'MegatronHook'], - 'priority': ['Priority', 'get'] + 'distributed.ddp_hook': ['DDPHook'], + 'distributed.deepspeed_hook': ['DeepspeedHook'], + 'distributed.megatron_hook': ['MegatronHook'], + 'priority': ['Priority', 'get_priority'] } import sys diff --git a/modelscope/trainers/hooks/distributed/__init__.py b/modelscope/trainers/hooks/distributed/__init__.py index 17b9c429..e69de29b 100644 --- a/modelscope/trainers/hooks/distributed/__init__.py +++ b/modelscope/trainers/hooks/distributed/__init__.py @@ -1,3 +0,0 @@ -from .ddp_hook import DDPHook -from .deepspeed_hook import DeepspeedHook -from .megatron_hook import MegatronHook From f6ea3eadea5c3e32d8a68374d5b7cd2763b82579 Mon Sep 17 00:00:00 2001 From: "chenyafeng.cyf" Date: Tue, 16 May 2023 22:28:20 +0800 Subject: [PATCH 46/76] eres2net Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12640199 --- modelscope/metainfo.py | 2 + modelscope/models/audio/sv/ERes2Net.py | 344 ++++++++++++++++++ modelscope/models/audio/sv/fusion.py | 32 ++ modelscope/models/audio/sv/pooling_layers.py | 107 ++++++ .../speaker_verification_eres2net_pipeline.py | 110 ++++++ tests/pipelines/test_speaker_verification.py | 11 + 6 files changed, 606 insertions(+) create mode 100644 modelscope/models/audio/sv/ERes2Net.py create mode 100644 modelscope/models/audio/sv/fusion.py create mode 100644 modelscope/models/audio/sv/pooling_layers.py create mode 100644 modelscope/pipelines/audio/speaker_verification_eres2net_pipeline.py diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index 2e3fa39d..c4057314 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -182,6 +182,7 @@ class Models(object): generic_sv = 'generic-sv' ecapa_tdnn_sv = 'ecapa-tdnn-sv' campplus_sv = 'cam++-sv' + eres2net_sv = 'eres2net-sv' scl_sd = 'scl-sd' rdino_tdnn_sv = 'rdino_ecapa-tdnn-sv' generic_lm = 'generic-lm' @@ -483,6 +484,7 @@ class Pipelines(object): vad_inference = 'vad-inference' speaker_verification = 'speaker-verification' speaker_verification_rdino = 'speaker-verification-rdino' + speaker_verification_eres2net = 'speaker-verification-eres2net' speaker_change_locating = 'speaker-change-locating' lm_inference = 'language-score-prediction' speech_timestamp_inference = 'speech-timestamp-inference' diff --git a/modelscope/models/audio/sv/ERes2Net.py b/modelscope/models/audio/sv/ERes2Net.py new file mode 100644 index 00000000..615be064 --- /dev/null +++ b/modelscope/models/audio/sv/ERes2Net.py @@ -0,0 +1,344 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +""" Res2Net implementation is adapted from https://github.com/wenet-e2e/wespeaker. + ERes2Net incorporates both local and global feature fusion techniques to improve the performance. The local feature + fusion (LFF) fuses the features within one single residual block to extract the local signal. + The global feature fusion (GFF) takes acoustic features of different scales as input to aggregate global signal. +""" +import math +import os +from typing import Any, Dict, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchaudio.compliance.kaldi as Kaldi + +import modelscope.models.audio.sv.pooling_layers as pooling_layers +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.models.audio.sv.fusion import AFF +from modelscope.utils.constant import Tasks + + +class ReLU(nn.Hardtanh): + + def __init__(self, inplace=False): + super(ReLU, self).__init__(0, 20, inplace) + + def __repr__(self): + inplace_str = 'inplace' if self.inplace else '' + return self.__class__.__name__ + ' (' \ + + inplace_str + ')' + + +def conv1x1(in_planes, out_planes, stride=1): + '1x1 convolution without padding' + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=1, + stride=stride, + padding=0, + bias=False) + + +def conv3x3(in_planes, out_planes, stride=1): + '3x3 convolution with padding' + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + + +class BasicBlockRes2Net(nn.Module): + expansion = 2 + + def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2): + super(BasicBlockRes2Net, self).__init__() + width = int(math.floor(planes * (baseWidth / 64.0))) + self.conv1 = conv1x1(in_planes, width * scale, stride) + self.bn1 = nn.BatchNorm2d(width * scale) + self.nums = scale + + convs = [] + bns = [] + for i in range(self.nums): + convs.append(conv3x3(width, width)) + bns.append(nn.BatchNorm2d(width)) + self.convs = nn.ModuleList(convs) + self.bns = nn.ModuleList(bns) + self.relu = ReLU(inplace=True) + + self.conv3 = conv1x1(width * scale, planes * self.expansion) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d( + in_planes, + self.expansion * planes, + kernel_size=1, + stride=stride, + bias=False), nn.BatchNorm2d(self.expansion * planes)) + self.stride = stride + self.width = width + self.scale = scale + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + spx = torch.split(out, self.width, 1) + for i in range(self.nums): + if i == 0: + sp = spx[i] + else: + sp = sp + spx[i] + sp = self.convs[i](sp) + sp = self.relu(self.bns[i](sp)) + if i == 0: + out = sp + else: + out = torch.cat((out, sp), 1) + + out = self.conv3(out) + out = self.bn3(out) + + residual = self.shortcut(x) + out += residual + out = self.relu(out) + + return out + + +class BasicBlockRes2Net_diff_AFF(nn.Module): + expansion = 2 + + def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2): + super(BasicBlockRes2Net_diff_AFF, self).__init__() + width = int(math.floor(planes * (baseWidth / 64.0))) + self.conv1 = conv1x1(in_planes, width * scale, stride) + self.bn1 = nn.BatchNorm2d(width * scale) + self.nums = scale + + convs = [] + fuse_models = [] + bns = [] + for i in range(self.nums): + convs.append(conv3x3(width, width)) + bns.append(nn.BatchNorm2d(width)) + for j in range(self.nums - 1): + fuse_models.append(AFF(channels=width)) + + self.convs = nn.ModuleList(convs) + self.bns = nn.ModuleList(bns) + self.fuse_models = nn.ModuleList(fuse_models) + self.relu = ReLU(inplace=True) + + self.conv3 = conv1x1(width * scale, planes * self.expansion) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d( + in_planes, + self.expansion * planes, + kernel_size=1, + stride=stride, + bias=False), nn.BatchNorm2d(self.expansion * planes)) + self.stride = stride + self.width = width + self.scale = scale + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + spx = torch.split(out, self.width, 1) + for i in range(self.nums): + if i == 0: + sp = spx[i] + else: + sp = self.fuse_models[i - 1](sp, spx[i]) + + sp = self.convs[i](sp) + sp = self.relu(self.bns[i](sp)) + if i == 0: + out = sp + else: + out = torch.cat((out, sp), 1) + + out = self.conv3(out) + out = self.bn3(out) + + residual = self.shortcut(x) + out += residual + out = self.relu(out) + + return out + + +class ERes2Net(nn.Module): + + def __init__(self, + block=BasicBlockRes2Net, + block_fuse=BasicBlockRes2Net_diff_AFF, + num_blocks=[3, 4, 6, 3], + m_channels=32, + feat_dim=80, + embed_dim=192, + pooling_func='TSTP', + two_emb_layer=False): + super(ERes2Net, self).__init__() + self.in_planes = m_channels + self.feat_dim = feat_dim + self.embed_dim = embed_dim + self.stats_dim = int(feat_dim / 8) * m_channels * 8 + self.two_emb_layer = two_emb_layer + + self.conv1 = nn.Conv2d( + 1, m_channels, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(m_channels) + self.layer1 = self._make_layer( + block, m_channels, num_blocks[0], stride=1) + self.layer2 = self._make_layer( + block, m_channels * 2, num_blocks[1], stride=2) + self.layer3 = self._make_layer( + block_fuse, m_channels * 4, num_blocks[2], stride=2) + self.layer4 = self._make_layer( + block_fuse, m_channels * 8, num_blocks[3], stride=2) + + # downsampling + self.layer1_downsample = nn.Conv2d( + m_channels * 2, + m_channels * 4, + kernel_size=3, + stride=2, + padding=1, + bias=False) + self.layer2_downsample = nn.Conv2d( + m_channels * 4, + m_channels * 8, + kernel_size=3, + padding=1, + stride=2, + bias=False) + self.layer3_downsample = nn.Conv2d( + m_channels * 8, + m_channels * 16, + kernel_size=3, + padding=1, + stride=2, + bias=False) + + # bottom-up fusion + self.fuse_mode12 = AFF(channels=m_channels * 4) + self.fuse_mode123 = AFF(channels=m_channels * 8) + self.fuse_mode1234 = AFF(channels=m_channels * 16) + + self.n_stats = 1 if pooling_func == 'TAP' or pooling_func == 'TSDP' else 2 + self.pool = getattr(pooling_layers, pooling_func)( + in_dim=self.stats_dim * block.expansion) + self.seg_1 = nn.Linear(self.stats_dim * block.expansion * self.n_stats, + embed_dim) + if self.two_emb_layer: + self.seg_bn_1 = nn.BatchNorm1d(embed_dim, affine=False) + self.seg_2 = nn.Linear(embed_dim, embed_dim) + else: + self.seg_bn_1 = nn.Identity() + self.seg_2 = nn.Identity() + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + x = x.permute(0, 2, 1) + + x = x.unsqueeze_(1) + out = F.relu(self.bn1(self.conv1(x))) + out1 = self.layer1(out) + + # bottom-up fusion + out2 = self.layer2(out1) + out1_downsample = self.layer1_downsample(out1) + fuse_out12 = self.fuse_mode12(out2, out1_downsample) + + out3 = self.layer3(out2) + fuse_out12_downsample = self.layer2_downsample(fuse_out12) + fuse_out123 = self.fuse_mode123(out3, fuse_out12_downsample) + + out4 = self.layer4(out3) + fuse_out123_downsample = self.layer3_downsample(fuse_out123) + fuse_out1234 = self.fuse_mode1234(out4, fuse_out123_downsample) + stats = self.pool(fuse_out1234) + + embed_a = self.seg_1(stats) + if self.two_emb_layer: + out = F.relu(embed_a) + out = self.seg_bn_1(out) + embed_b = self.seg_2(out) + return embed_b + else: + return embed_a + + +@MODELS.register_module( + Tasks.speaker_verification, module_name=Models.eres2net_sv) +class SpeakerVerificationERes2Net(TorchModel): + r"""Enhanced Res2Net architecture with local and global feature fusion. ERes2Net is mainly composed + of LFF and GFF. The LFF extracts localization-preserved speaker features and strengthen the local information + interaction. GFF fuses multi-scale feature maps in bottom-up pathway to obtain global information. + Args: + model_dir: A model dir. + model_config: The model config. + """ + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + self.other_config = kwargs + self.feature_dim = 80 + + self.embedding_model = ERes2Net() + + pretrained_model_name = kwargs['pretrained_model'] + self.__load_check_point(pretrained_model_name) + + self.embedding_model.eval() + + def forward(self, audio): + assert len(audio.shape) == 2 and audio.shape[ + 0] == 1, 'modelscope error: the shape of input audio to model needs to be [1, T]' + # audio shape: [1, T] + feature = self.__extract_feature(audio) + embedding = self.embedding_model(feature) + + return embedding + + def __extract_feature(self, audio): + feature = Kaldi.fbank(audio, num_mel_bins=self.feature_dim) + feature = feature - feature.mean(dim=0, keepdim=True) + feature = feature.unsqueeze(0) + return feature + + def __load_check_point(self, pretrained_model_name, device=None): + if not device: + device = torch.device('cpu') + self.embedding_model.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_model_name), + map_location=device), + strict=True) diff --git a/modelscope/models/audio/sv/fusion.py b/modelscope/models/audio/sv/fusion.py new file mode 100644 index 00000000..615529bd --- /dev/null +++ b/modelscope/models/audio/sv/fusion.py @@ -0,0 +1,32 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn + + +class AFF(nn.Module): + + def __init__(self, channels=64, r=4): + super(AFF, self).__init__() + inter_channels = int(channels // r) + + self.local_att = nn.Sequential( + nn.Conv2d( + channels * 2, + inter_channels, + kernel_size=1, + stride=1, + padding=0), + nn.BatchNorm2d(inter_channels), + nn.SiLU(inplace=True), + nn.Conv2d( + inter_channels, channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm2d(channels), + ) + + def forward(self, x, ds_y): + xa = torch.cat((x, ds_y), dim=1) + x_att = self.local_att(xa) + x_att = 1.0 + torch.tanh(x_att) + xo = torch.mul(x, x_att) + torch.mul(ds_y, 2.0 - x_att) + + return xo diff --git a/modelscope/models/audio/sv/pooling_layers.py b/modelscope/models/audio/sv/pooling_layers.py new file mode 100644 index 00000000..0fdc44ca --- /dev/null +++ b/modelscope/models/audio/sv/pooling_layers.py @@ -0,0 +1,107 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +""" This implementation is adapted from https://github.com/wenet-e2e/wespeaker. +""" +import torch +import torch.nn as nn + + +class TAP(nn.Module): + """ + Temporal average pooling, only first-order mean is considered + """ + + def __init__(self, **kwargs): + super(TAP, self).__init__() + + def forward(self, x): + pooling_mean = x.mean(dim=-1) + # To be compatable with 2D input + pooling_mean = pooling_mean.flatten(start_dim=1) + return pooling_mean + + +class TSDP(nn.Module): + """ + Temporal standard deviation pooling, only second-order std is considered + """ + + def __init__(self, **kwargs): + super(TSDP, self).__init__() + + def forward(self, x): + # The last dimension is the temporal axis + pooling_std = torch.sqrt(torch.var(x, dim=-1) + 1e-8) + pooling_std = pooling_std.flatten(start_dim=1) + return pooling_std + + +class TSTP(nn.Module): + """ + Temporal statistics pooling, concatenate mean and std, which is used in + x-vector + Comment: simple concatenation can not make full use of both statistics + """ + + def __init__(self, **kwargs): + super(TSTP, self).__init__() + + def forward(self, x): + # The last dimension is the temporal axis + pooling_mean = x.mean(dim=-1) + pooling_std = torch.sqrt(torch.var(x, dim=-1) + 1e-8) + pooling_mean = pooling_mean.flatten(start_dim=1) + pooling_std = pooling_std.flatten(start_dim=1) + + stats = torch.cat((pooling_mean, pooling_std), 1) + return stats + + +class ASTP(nn.Module): + """ Attentive statistics pooling: Channel- and context-dependent + statistics pooling, first used in ECAPA_TDNN. + """ + + def __init__(self, in_dim, bottleneck_dim=128, global_context_att=False): + super(ASTP, self).__init__() + self.global_context_att = global_context_att + + # Use Conv1d with stride == 1 rather than Linear, then we don't + # need to transpose inputs. + if global_context_att: + self.linear1 = nn.Conv1d( + in_dim * 3, bottleneck_dim, + kernel_size=1) # equals W and b in the paper + else: + self.linear1 = nn.Conv1d( + in_dim, bottleneck_dim, + kernel_size=1) # equals W and b in the paper + self.linear2 = nn.Conv1d( + bottleneck_dim, in_dim, + kernel_size=1) # equals V and k in the paper + + def forward(self, x): + """ + x: a 3-dimensional tensor in tdnn-based architecture (B,F,T) + or a 4-dimensional tensor in resnet architecture (B,C,F,T) + 0-dim: batch-dimension, last-dim: time-dimension (frame-dimension) + """ + if len(x.shape) == 4: + x = x.reshape(x.shape[0], x.shape[1] * x.shape[2], x.shape[3]) + assert len(x.shape) == 3 + + if self.global_context_att: + context_mean = torch.mean(x, dim=-1, keepdim=True).expand_as(x) + context_std = torch.sqrt( + torch.var(x, dim=-1, keepdim=True) + 1e-10).expand_as(x) + x_in = torch.cat((x, context_mean, context_std), dim=1) + else: + x_in = x + + # DON'T use ReLU here! ReLU may be hard to converge. + alpha = torch.tanh( + self.linear1(x_in)) # alpha = F.relu(self.linear1(x_in)) + alpha = torch.softmax(self.linear2(alpha), dim=2) + mean = torch.sum(alpha * x, dim=2) + var = torch.sum(alpha * (x**2), dim=2) - mean**2 + std = torch.sqrt(var.clamp(min=1e-10)) + return torch.cat([mean, std], dim=1) diff --git a/modelscope/pipelines/audio/speaker_verification_eres2net_pipeline.py b/modelscope/pipelines/audio/speaker_verification_eres2net_pipeline.py new file mode 100644 index 00000000..ef91d83b --- /dev/null +++ b/modelscope/pipelines/audio/speaker_verification_eres2net_pipeline.py @@ -0,0 +1,110 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import io +from typing import Any, Dict, List, Union + +import soundfile as sf +import torch + +from modelscope.fileio import File +from modelscope.metainfo import Pipelines +from modelscope.outputs import OutputKeys +from modelscope.pipelines.base import InputModel, Pipeline +from modelscope.pipelines.builder import PIPELINES +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@PIPELINES.register_module( + Tasks.speaker_verification, + module_name=Pipelines.speaker_verification_eres2net) +class ERes2Net_Pipeline(Pipeline): + """Speaker Verification Inference Pipeline + use `model` to create a Speaker Verification pipeline. + + Args: + model (SpeakerVerificationPipeline): A model instance, or a model local dir, or a model id in the model hub. + kwargs (dict, `optional`): + Extra kwargs passed into the pipeline's constructor. + Example: + >>> from modelscope.pipelines import pipeline + >>> from modelscope.utils.constant import Tasks + >>> p = pipeline( + >>> task=Tasks.speaker_verification, model='damo/speech_ecapa-tdnn_sv_en_voxceleb_16k') + >>> print(p([audio_1, audio_2])) + + """ + + def __init__(self, model: InputModel, **kwargs): + """use `model` to create a speaker verification pipeline for prediction + Args: + model (str): a valid offical model id + """ + super().__init__(model=model, **kwargs) + self.model_config = self.model.model_config + self.config = self.model.other_config + self.thr = self.config['yesOrno_thr'] + + def __call__(self, + in_audios: List[str], + thr: float = None) -> Dict[str, Any]: + if thr is not None: + self.thr = thr + if self.thr < -1 or self.thr > 1: + raise ValueError( + 'modelscope error: the thr value should be in [-1, 1], but found to be %f.' + % self.thr) + outputs = self.preprocess(in_audios) + outputs = self.forward(outputs) + outputs = self.postprocess(outputs) + + return outputs + + def forward(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + emb1 = self.model(inputs['data1']) + emb2 = self.model(inputs['data2']) + + return {'emb1': emb1, 'emb2': emb2} + + def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + score = self.compute_cos_similarity(inputs['emb1'], inputs['emb2']) + score = round(score, 5) + if score >= self.thr: + ans = 'yes' + else: + ans = 'no' + + return {OutputKeys.SCORE: score, OutputKeys.TEXT: ans} + + def preprocess(self, inputs: List[str], + **preprocess_params) -> Dict[str, Any]: + if len(inputs) != 2: + raise ValueError( + 'modelscope error: Two input audio files are required.') + output = {} + for i in range(len(inputs)): + if isinstance(inputs[i], str): + file_bytes = File.read(inputs[i]) + data, fs = sf.read(io.BytesIO(file_bytes), dtype='float32') + if len(data.shape) == 2: + data = data[:, 0] + if fs != self.model_config['sample_rate']: + raise ValueError( + 'modelscope error: Only support %d sample rate files' + % self.model_cfg['sample_rate']) + output['data%d' % + (i + 1)] = torch.from_numpy(data).unsqueeze(0) + else: + raise ValueError( + 'modelscope error: The input type is temporarily restricted to audio file address' + % i) + return output + + def compute_cos_similarity(self, emb1: torch.Tensor, + emb2: torch.Tensor) -> float: + assert len(emb1.shape) == 2 and len(emb2.shape) == 2 + cos = torch.nn.CosineSimilarity(dim=1, eps=1e-6) + cosine = cos(emb1, emb2) + return cosine.item() diff --git a/tests/pipelines/test_speaker_verification.py b/tests/pipelines/test_speaker_verification.py index ae498163..2b90c66e 100644 --- a/tests/pipelines/test_speaker_verification.py +++ b/tests/pipelines/test_speaker_verification.py @@ -22,6 +22,7 @@ class SpeakerVerificationTest(unittest.TestCase): campplus_voxceleb_16k_model_id = 'damo/speech_campplus_sv_en_voxceleb_16k' rdino_voxceleb_16k_model_id = 'damo/speech_rdino_ecapa_tdnn_sv_en_voxceleb_16k' speaker_change_locating_cn_model_id = 'damo/speech_campplus-transformer_scl_zh-cn_16k-common' + eres2net_voxceleb_16k_model_id = 'damo/speech_eres2net_sv_en_voxceleb_16k' def setUp(self) -> None: self.task = Tasks.speaker_verification @@ -80,6 +81,16 @@ class SpeakerVerificationTest(unittest.TestCase): print(result) self.assertTrue(OutputKeys.TEXT in result) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_speaker_verification_eres2net_voxceleb_16k(self): + logger.info('Run speaker verification for eres2net_voxceleb_16k model') + result = self.run_pipeline( + model_id=self.eres2net_voxceleb_16k_model_id, + audios=[SPEAKER1_A_EN_16K_WAV, SPEAKER1_B_EN_16K_WAV], + model_revision='v1.0.2') + print(result) + self.assertTrue(OutputKeys.SCORE in result) + if __name__ == '__main__': unittest.main() From 6abdce929d5000ad60d2e993336ec6cf567b9db2 Mon Sep 17 00:00:00 2001 From: "mulin.lyh" Date: Wed, 17 May 2023 22:11:23 +0800 Subject: [PATCH 47/76] fix py38 issue Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12660340 --- modelscope/models/cv/object_detection_3d/depe/result_vis.py | 2 +- modelscope/pipelines/cv/image_detection_pipeline.py | 2 +- .../cv/referring_video_object_segmentation_pipeline.py | 2 +- requirements/audio/audio_signal.txt | 2 +- requirements/audio/audio_tts.txt | 2 +- requirements/cv.txt | 2 +- requirements/framework.txt | 3 ++- requirements/multi-modal.txt | 2 +- tests/run_config.yaml | 6 ++++++ 9 files changed, 15 insertions(+), 8 deletions(-) diff --git a/modelscope/models/cv/object_detection_3d/depe/result_vis.py b/modelscope/models/cv/object_detection_3d/depe/result_vis.py index d567654e..efaef0b5 100644 --- a/modelscope/models/cv/object_detection_3d/depe/result_vis.py +++ b/modelscope/models/cv/object_detection_3d/depe/result_vis.py @@ -169,7 +169,7 @@ def plot_result(res_path, check_point_in_img(corners_img, img.shape[0], img.shape[1])) valid = valid.reshape( -1, 8) # valid means: d>0 and visible in current view - corners_img = corners_img.reshape(-1, 8, 2).astype(np.int) + corners_img = corners_img.reshape(-1, 8, 2).astype(int) for aid in range(valid.shape[0]): if scores[aid] < vis_thred and pred_flag[aid]: continue diff --git a/modelscope/pipelines/cv/image_detection_pipeline.py b/modelscope/pipelines/cv/image_detection_pipeline.py index 86963c37..2b8275c2 100644 --- a/modelscope/pipelines/cv/image_detection_pipeline.py +++ b/modelscope/pipelines/cv/image_detection_pipeline.py @@ -30,7 +30,7 @@ class ImageDetectionPipeline(Pipeline): def preprocess(self, input: Input) -> Dict[str, Any]: img = LoadImage.convert_to_ndarray(input) - img = img.astype(np.float) + img = img.astype(np.float64) img = self.model.preprocess(img) result = {'img': img} return result diff --git a/modelscope/pipelines/cv/referring_video_object_segmentation_pipeline.py b/modelscope/pipelines/cv/referring_video_object_segmentation_pipeline.py index 39195bcd..123057f5 100644 --- a/modelscope/pipelines/cv/referring_video_object_segmentation_pipeline.py +++ b/modelscope/pipelines/cv/referring_video_object_segmentation_pipeline.py @@ -225,7 +225,7 @@ class ReferringVideoObjectSegmentationPipeline(Pipeline): def apply_mask(image, mask, color, transparency=0.7): mask = mask[..., np.newaxis].repeat(repeats=3, axis=2) mask = mask * transparency - color_matrix = np.ones(image.shape, dtype=np.float) * color + color_matrix = np.ones(image.shape, dtype=np.float64) * color out_image = color_matrix * mask + image * (1.0 - mask) return out_image diff --git a/requirements/audio/audio_signal.txt b/requirements/audio/audio_signal.txt index 71037baa..16a18e67 100644 --- a/requirements/audio/audio_signal.txt +++ b/requirements/audio/audio_signal.txt @@ -1,5 +1,5 @@ hyperpyyaml -librosa<=0.9.2 +librosa==0.9.2 MinDAEC mir_eval>=0.7 numpy diff --git a/requirements/audio/audio_tts.txt b/requirements/audio/audio_tts.txt index b1a85faf..81a5c6f4 100644 --- a/requirements/audio/audio_tts.txt +++ b/requirements/audio/audio_tts.txt @@ -3,7 +3,7 @@ greenlet>=1.1.2 inflect jedi>=0.18.1 kantts -librosa<=0.9.2 +librosa==0.9.2 lxml matplotlib msgpack>=1.0.4 diff --git a/requirements/cv.txt b/requirements/cv.txt index 6ea02c54..0cec3659 100644 --- a/requirements/cv.txt +++ b/requirements/cv.txt @@ -6,7 +6,7 @@ chumpy clip>=1.0 control_ldm ddpm_guided_diffusion -diffusers +diffusers>=0.13.1,<0.15.0 easydict easyrobust edit_distance diff --git a/requirements/framework.txt b/requirements/framework.txt index 03edde4d..10a1ef63 100644 --- a/requirements/framework.txt +++ b/requirements/framework.txt @@ -4,7 +4,8 @@ datasets>=2.7.0,<=2.8.0 einops filelock>=3.3.0 gast>=0.2.2 -numpy<1.24.0 +# for python3.7 python3.8 compatible +numpy<1.22.0 oss2 # for datasets compatible pandas<=1.5.3 diff --git a/requirements/multi-modal.txt b/requirements/multi-modal.txt index b07b99a2..9d2c3448 100644 --- a/requirements/multi-modal.txt +++ b/requirements/multi-modal.txt @@ -1,7 +1,7 @@ accelerate diffusers>=0.13.1,<0.15.0 ftfy>=6.0.3 -librosa<=0.9.2 +librosa==0.9.2 opencv-python pycocoevalcap>=1.2 pycocotools>=2.0.4 diff --git a/tests/run_config.yaml b/tests/run_config.yaml index 14de4c85..c3daf065 100644 --- a/tests/run_config.yaml +++ b/tests/run_config.yaml @@ -87,3 +87,9 @@ envs: - test_image_style_transfer.py - test_image_portrait_stylization_trainer.py - test_language_identification.py + - test_ocr_detection_db_trainer.py + - test_nerf_recon_acc.py + - test_tensorboard_hook.py + - test_efficient_diffusion_tuning_trainer.py + - test_trainer.py + - test_nerf_recon_acc_trainer.py From d38fa563a8111f5bf31d095aece8c9ee309a3418 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Fri, 19 May 2023 10:48:33 +0800 Subject: [PATCH 48/76] Re commit lossing files for flextrain Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12674746 --- .../finetune_text_generation.py | 22 ++++++++++++++----- .../pytorch/text_generation/run_train_gpt3.sh | 3 +++ .../pytorch/text_generation/run_train_mt5.sh | 3 +++ .../pytorch/text_generation/run_train_palm.sh | 3 +++ modelscope/trainers/trainer.py | 2 +- modelscope/trainers/training_args.py | 7 +++--- 6 files changed, 31 insertions(+), 9 deletions(-) diff --git a/examples/pytorch/text_generation/finetune_text_generation.py b/examples/pytorch/text_generation/finetune_text_generation.py index a89970e8..1631911e 100644 --- a/examples/pytorch/text_generation/finetune_text_generation.py +++ b/examples/pytorch/text_generation/finetune_text_generation.py @@ -1,6 +1,7 @@ from dataclasses import dataclass, field -from modelscope import EpochBasedTrainer, MsDataset, TrainingArgs +from modelscope import (EpochBasedTrainer, MsDataset, TrainingArgs, + build_dataset_from_file) from modelscope.metainfo import Trainers from modelscope.trainers import build_trainer @@ -94,14 +95,25 @@ def cfg_modify_fn(cfg): return cfg -dataset = MsDataset.load(args.train_dataset_name) -train_dataset = dataset['train'] -eval_dataset = dataset['validation' if 'validation' in dataset else 'test'] +if args.dataset_json_file is None: + train_dataset = MsDataset.load( + args.train_dataset_name, + subset_name=args.train_subset_name, + split=args.train_split, + namespace=args.train_dataset_namespace) + validation_dataset = MsDataset.load( + args.val_dataset_name, + subset_name=args.val_subset_name, + split=args.val_split, + namespace=args.val_dataset_namespace) +else: + train_dataset, validation_dataset = build_dataset_from_file( + args.dataset_json_file) kwargs = dict( model=args.model, train_dataset=train_dataset, - eval_dataset=eval_dataset, + eval_dataset=validation_dataset, seed=args.seed, work_dir=args.work_dir, cfg_modify_fn=cfg_modify_fn) diff --git a/examples/pytorch/text_generation/run_train_gpt3.sh b/examples/pytorch/text_generation/run_train_gpt3.sh index fd37b42c..78521951 100644 --- a/examples/pytorch/text_generation/run_train_gpt3.sh +++ b/examples/pytorch/text_generation/run_train_gpt3.sh @@ -9,6 +9,9 @@ PYTHONPATH=. torchrun --nproc_per_node $WORLD_SIZE examples/pytorch/text_generat --work_dir './tmp' \ --model 'damo/nlp_gpt3_text-generation_1.3B' \ --train_dataset_name 'chinese-poetry-collection' \ + --val_dataset_name 'chinese-poetry-collection' \ + --train_split 'train' \ + --val_split 'test' \ --preprocessor 'text-gen-jieba-tokenizer' \ --src_txt 'text1' \ --tgt_txt 'text2' \ diff --git a/examples/pytorch/text_generation/run_train_mt5.sh b/examples/pytorch/text_generation/run_train_mt5.sh index 6d032d6e..b2d0bbf1 100644 --- a/examples/pytorch/text_generation/run_train_mt5.sh +++ b/examples/pytorch/text_generation/run_train_mt5.sh @@ -4,6 +4,9 @@ PYTHONPATH=. torchrun examples/pytorch/text_generation/finetune_text_generation. --task 'text2text-generation' \ --model 'damo/nlp_mt5_zero-shot-augment_chinese-base' \ --train_dataset_name 'DuReader_robust-QG' \ + --val_dataset_name 'DuReader_robust-QG' \ + --train_split 'train' \ + --val_split 'validation' \ --src_txt 'text1' \ --tgt_txt 'text2' \ --max_epochs 1 \ diff --git a/examples/pytorch/text_generation/run_train_palm.sh b/examples/pytorch/text_generation/run_train_palm.sh index 68b9e89d..06153812 100644 --- a/examples/pytorch/text_generation/run_train_palm.sh +++ b/examples/pytorch/text_generation/run_train_palm.sh @@ -3,6 +3,9 @@ PYTHONPATH=. torchrun examples/pytorch/text_generation/finetune_text_generation. --work_dir './tmp' \ --model 'damo/nlp_palm2.0_pretrained_chinese-base' \ --train_dataset_name 'DuReader_robust-QG' \ + --val_dataset_name 'DuReader_robust-QG' \ + --train_split 'train' \ + --val_split 'validation' \ --src_txt 'text1' \ --tgt_txt 'text2' \ --max_epochs 1 \ diff --git a/modelscope/trainers/trainer.py b/modelscope/trainers/trainer.py index c980de04..278d6b44 100644 --- a/modelscope/trainers/trainer.py +++ b/modelscope/trainers/trainer.py @@ -155,10 +155,10 @@ class EpochBasedTrainer(BaseTrainer): self.cfg_modify_fn = cfg_modify_fn # add default config merge_cfg(self.cfg) - self.cfg = self.rebuild_config(self.cfg) if 'cfg_options' in kwargs: self.cfg.merge_from_dict(kwargs['cfg_options']) self.cfg = update_cfg(self.cfg) + self.cfg = self.rebuild_config(self.cfg) if isinstance(model, (TorchModel, nn.Module)): self.model = model diff --git a/modelscope/trainers/training_args.py b/modelscope/trainers/training_args.py index b7236163..16453d7c 100644 --- a/modelscope/trainers/training_args.py +++ b/modelscope/trainers/training_args.py @@ -9,6 +9,7 @@ import json from modelscope.trainers.cli_argument_parser import CliArgumentParser from modelscope.utils.config import Config +from modelscope.utils.constant import DEFAULT_DATASET_NAMESPACE def set_flatten_value(values: Union[str, List[str]]): @@ -62,13 +63,13 @@ class DatasetArgs: }) train_dataset_namespace: str = field( - default=None, + default=DEFAULT_DATASET_NAMESPACE, metadata={ 'help': 'The dataset namespace used for training', }) val_dataset_namespace: str = field( - default=None, + default=DEFAULT_DATASET_NAMESPACE, metadata={ 'help': 'The dataset namespace used for evaluating', }) @@ -450,7 +451,7 @@ class TrainingArgs(DatasetArgs, TrainArgs, ModelArgs): _unknown[unknown[i].replace('-', '')] = parse_value(unknown[i + 1]) args_dict = vars(args) self.manual_args += parser.manual_args - + self._unknown_args.update(_unknown) for key, value in deepcopy(args_dict).items(): if key is not None and hasattr(self, key): setattr(self, key, value) From f2640a5a12a0fc506be184c26f5ad7a9e53090dd Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Wed, 24 May 2023 19:48:20 +0800 Subject: [PATCH 49/76] fix private dataset auth issue 1. Fix private datasets auth issue 2. Add arg: token (optional) in MsDataset.load() for FlexTrain Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12721569 --- modelscope/hub/api.py | 36 +++++++ .../msdatasets/data_loader/data_loader.py | 1 - .../data_loader/data_loader_manager.py | 6 +- .../msdatasets/download/dataset_builder.py | 100 +++++++++++++++++- modelscope/msdatasets/ms_dataset.py | 7 ++ tests/msdatasets/test_ms_dataset.py | 3 +- 6 files changed, 143 insertions(+), 10 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index e3436aea..75e02b9d 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -16,6 +16,7 @@ from http.cookiejar import CookieJar from os.path import expanduser from typing import Dict, List, Optional, Tuple, Union +import pandas as pd import requests from requests import Session from requests.adapters import HTTPAdapter, Retry @@ -651,6 +652,41 @@ class HubApi: return text_headers, text_content + @staticmethod + def fetch_csv_from_url(url, out_path, chunk_size=1000000): + from io import StringIO + import hashlib + out_path = os.path.join(out_path, hashlib.md5(url.encode(encoding='UTF-8')).hexdigest()) + if os.path.exists(out_path): + return out_path + cookies = ModelScopeConfig.get_cookies() + + # Make the request and get the response content as TextIO + response = requests.get(url, cookies=cookies) + data = StringIO(response.text) + + # Use read_csv with the TextIO object + csv_file_reader = pd.read_csv(data, iterator=True, dtype=str, delimiter=None) + + loop = True + iter_num = 0 + while loop: + try: + chunk = csv_file_reader.get_chunk(size=chunk_size) + + if iter_num == 0: + with_header = True + else: + with_header = False + + chunk.to_csv(out_path, mode='a', index=False, header=with_header) + iter_num += 1 + except StopIteration: + loop = False + logger.info('stop chunk iteration') + + return out_path + def get_dataset_file_url( self, file_name: str, diff --git a/modelscope/msdatasets/data_loader/data_loader.py b/modelscope/msdatasets/data_loader/data_loader.py index b1450c61..f29acc8f 100644 --- a/modelscope/msdatasets/data_loader/data_loader.py +++ b/modelscope/msdatasets/data_loader/data_loader.py @@ -86,7 +86,6 @@ class OssDownloader(BaseDownloader): def _authorize(self) -> None: """ Authorization of target dataset. Get credentials from cache and send to the modelscope-hub in the future. """ - # TODO: obtain credentials from loacl cache when available. cookies = ModelScopeConfig.get_cookies() git_token = ModelScopeConfig.get_token() user_info = ModelScopeConfig.get_user_info() diff --git a/modelscope/msdatasets/data_loader/data_loader_manager.py b/modelscope/msdatasets/data_loader/data_loader_manager.py index 5be32de1..0dec5d89 100644 --- a/modelscope/msdatasets/data_loader/data_loader_manager.py +++ b/modelscope/msdatasets/data_loader/data_loader_manager.py @@ -127,15 +127,15 @@ class RemoteDataLoaderManager(DataLoaderManager): return dataset_ret # To use the modelscope data loader elif data_loader_type == RemoteDataLoaderType.MS_DATA_LOADER: - oss_data_loader = OssDownloader( + oss_downloader = OssDownloader( dataset_context_config=self.dataset_context_config) - oss_data_loader.process() + oss_downloader.process() # download statistics self.api.dataset_download_statistics( dataset_name=dataset_name, namespace=namespace, use_streaming=use_streaming) - return oss_data_loader.dataset + return oss_downloader.dataset else: raise f'Expected remote data loader type: {RemoteDataLoaderType.HF_DATA_LOADER.value}/' \ f'{RemoteDataLoaderType.MS_DATA_LOADER.value}, but got {data_loader_type} .' diff --git a/modelscope/msdatasets/download/dataset_builder.py b/modelscope/msdatasets/download/dataset_builder.py index 8ad5243a..b3ed5a8e 100644 --- a/modelscope/msdatasets/download/dataset_builder.py +++ b/modelscope/msdatasets/download/dataset_builder.py @@ -6,8 +6,9 @@ from typing import Dict, Union import datasets import pandas as pd import pyarrow as pa -from datasets import (ArrowBasedBuilder, GeneratorBasedBuilder, - IterableDataset, IterableDatasetDict) +from datasets import (ArrowBasedBuilder, Dataset, DatasetDict, + GeneratorBasedBuilder, IterableDataset, + IterableDatasetDict) from datasets.filesystems import is_remote_filesystem from datasets.info import DatasetInfo from datasets.naming import camelcase_to_snakecase @@ -47,6 +48,7 @@ class CsvDatasetBuilder(csv.Csv): self.meta_data_files = dataset_context_config.data_meta_config.meta_data_files self.zip_data_files = dataset_context_config.data_meta_config.zip_data_files self.input_config_kwargs = dataset_context_config.config_kwargs + self.split_path_dict = dict({}) self.cache_build_dir = os.path.join(self.cache_root_dir, self.namespace, self.dataset_name, @@ -61,16 +63,25 @@ class CsvDatasetBuilder(csv.Csv): sub_dir_hash = get_subdir_hash_from_split( split=split, version=self.version) + from datasets.data_files import DataFilesDict, DataFilesList + data_files = { + k: DataFilesList([v], origin_metadata=None) + for k, v in self.meta_data_files.items() + } + data_files = DataFilesDict.from_local_or_remote(data_files) + super().__init__( cache_dir=self.cache_build_dir, config_name=self.namespace, hash=sub_dir_hash, - data_files=self.meta_data_files, + data_files=data_files, **self.input_config_kwargs) self.info.builder_name = self.dataset_name self.name = camelcase_to_snakecase(self.dataset_name) + self.local_meta_csv_paths: dict = dict({}) + def _build_cache_dir(self, namespace=DEFAULT_DATASET_NAMESPACE): builder_data_dir = os.path.join( self._cache_dir_root, @@ -147,6 +158,87 @@ class CsvDatasetBuilder(csv.Csv): f"Failed to read file '{file}' with error {type(e)}: {e}") raise + def download_and_prepare(self, download_mode, dl_manager, + **download_kwargs): + + target_cache_dir = dl_manager.download_config.cache_dir + + split_name = dl_manager.download_config.split + if not split_name: + split_name = DatasetPathName.LOCK_FILE_NAME_ANY + version_name = dl_manager.download_config.version + if not version_name: + version_name = DatasetPathName.LOCK_FILE_NAME_ANY + subset_name = self.subset_name + if not subset_name: + subset_name = DatasetPathName.LOCK_FILE_NAME_ANY + + # Prevent parallel disk operations + lock_file_names = [] + lock_file_names.append(DatasetPathName.DATA_FILES_NAME) + lock_file_names.append(dl_manager.download_config.dataset_name) + lock_file_names.append(version_name) + lock_file_names.append(subset_name) + lock_file_names.append(split_name) + + lock_file_name = DatasetPathName.LOCK_FILE_NAME_DELIMITER.join( + lock_file_names) + + lock_path = os.path.join( + target_cache_dir.strip(DatasetPathName.DATA_FILES_NAME), + lock_file_name + '.lock') + with FileLock(lock_path): + data_exists = os.path.exists(target_cache_dir) + if data_exists and download_mode == DownloadMode.REUSE_DATASET_IF_EXISTS.value: + logger.warning( + f'Reusing dataset {self.name} ({target_cache_dir})') + logger.info(f'Generating dataset {self.name} ({target_cache_dir})') + + self._download_and_prepare( + dl_manager=dl_manager, download_mode=download_mode) + + def _download_and_prepare(self, dl_manager, download_mode): + import shutil + + target_cache_dir = dl_manager.download_config.cache_dir + if download_mode == DownloadMode.FORCE_REDOWNLOAD.value: + shutil.rmtree(target_cache_dir, ignore_errors=True) + os.makedirs(target_cache_dir, exist_ok=True) + + self.local_meta_csv_paths = { + k: HubApi.fetch_csv_from_url(v, target_cache_dir) + for k, v in self.meta_data_files.items() + } + + self.split_path_dict = dl_manager.download_and_extract( + self.zip_data_files) + + def _convert_csv_to_dataset(self, split_name, csv_file_path): + + df = pd.read_csv( + csv_file_path, iterator=False, delimiter=self.csv_delimiter) + + transform_fields = [] + for field_name in df.columns.tolist(): + if field_name.endswith(':FILE'): + transform_fields.append(field_name) + + base_extracted_dir = self.split_path_dict.get(split_name, '') + for field_name in transform_fields: + if base_extracted_dir: + df[field_name] = df[field_name].apply( + lambda x: os.path.join(base_extracted_dir, x)) + + pa_data = pa.Table.from_pandas(df) + return Dataset(arrow_table=pa_data) + + def as_dataset(self) -> DatasetDict: + + return DatasetDict({ + k: self._convert_csv_to_dataset(k, v) + for k, v in self.local_meta_csv_paths.items() + }) + class TaskSpecificDatasetBuilder(CsvDatasetBuilder): @@ -181,7 +273,7 @@ class TaskSpecificDatasetBuilder(CsvDatasetBuilder): self._cache_dir.replace(os.sep, '_') + '.lock') with FileLock(lock_path): data_exists = os.path.exists(self._cache_dir) - if data_exists and download_mode == DownloadMode.REUSE_DATASET_IF_EXISTS: + if data_exists and download_mode == DownloadMode.REUSE_DATASET_IF_EXISTS: # TODO: .value?? logger.warning( f'Reusing dataset {self.name} ({self._cache_dir})') return diff --git a/modelscope/msdatasets/ms_dataset.py b/modelscope/msdatasets/ms_dataset.py index 912e061d..fa5bac8d 100644 --- a/modelscope/msdatasets/ms_dataset.py +++ b/modelscope/msdatasets/ms_dataset.py @@ -170,6 +170,7 @@ class MsDataset: cache_dir: Optional[str] = MS_DATASETS_CACHE, use_streaming: Optional[bool] = False, custom_cfg: Optional[Config] = Config(), + token: Optional[str] = None, **config_kwargs, ) -> Union[dict, 'MsDataset', NativeIterableDataset]: """Load a MsDataset from the ModelScope Hub, Hugging Face Hub, urls, or a local dataset. @@ -197,12 +198,18 @@ class MsDataset: NativeIterableDataset or a dict of NativeIterableDataset. custom_cfg (str, Optional): Model configuration, this can be used for custom datasets. see https://modelscope.cn/docs/Configuration%E8%AF%A6%E8%A7%A3 + token (str, Optional): SDK token of ModelScope. **config_kwargs (additional keyword arguments): Keyword arguments to be passed Returns: MsDataset (MsDataset): MsDataset object for a certain dataset. """ + if token: + from modelscope.hub.api import HubApi + api = HubApi() + api.login(token) + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) hub = Hubs(hub or Hubs.modelscope) diff --git a/tests/msdatasets/test_ms_dataset.py b/tests/msdatasets/test_ms_dataset.py index ddb84b45..83ffd3f8 100644 --- a/tests/msdatasets/test_ms_dataset.py +++ b/tests/msdatasets/test_ms_dataset.py @@ -241,8 +241,7 @@ class MsDatasetTest(unittest.TestCase): use_streaming=True) data_example = next(iter(dataset)) print(data_example) - assert isinstance(data_example['Noisy Image:FILE:Object'], - PngImageFile) + assert data_example.values() @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_to_ms_dataset(self): From a524e01e474696543ca1a74dfb59e90279ec5b73 Mon Sep 17 00:00:00 2001 From: "lee.lcy" Date: Thu, 25 May 2023 10:44:09 +0800 Subject: [PATCH 50/76] feat: support onnx export for domain_specific_object_detection Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12682688 --- .../exporters/cv/object_detection_damoyolo_exporter.py | 3 +++ tests/export/test_export_object_detection_damoyolo.py | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/modelscope/exporters/cv/object_detection_damoyolo_exporter.py b/modelscope/exporters/cv/object_detection_damoyolo_exporter.py index 673811ad..6f271d39 100644 --- a/modelscope/exporters/cv/object_detection_damoyolo_exporter.py +++ b/modelscope/exporters/cv/object_detection_damoyolo_exporter.py @@ -13,6 +13,9 @@ from modelscope.metainfo import Models from modelscope.utils.constant import ModelFile, Tasks +@EXPORTERS.register_module( + Tasks.domain_specific_object_detection, + module_name=Models.tinynas_damoyolo) @EXPORTERS.register_module( Tasks.image_object_detection, module_name=Models.tinynas_damoyolo) class ObjectDetectionDamoyoloExporter(TorchModelExporter): diff --git a/tests/export/test_export_object_detection_damoyolo.py b/tests/export/test_export_object_detection_damoyolo.py index d7e51165..9a810e93 100644 --- a/tests/export/test_export_object_detection_damoyolo.py +++ b/tests/export/test_export_object_detection_damoyolo.py @@ -27,6 +27,15 @@ class TestExportObjectDetectionDamoyolo(unittest.TestCase): Exporter.from_model(model).export_onnx( input_shape=(1, 3, 640, 640), output_dir=self.tmp_dir) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_export_domain_specific_object_detection_damoyolo(self): + + model_id = 'damo/cv_tinynas_human-detection_damoyolo' + model = Model.from_pretrained(model_id) + with tempfile.TemporaryDirectory() as tmp_dir: + Exporter.from_model(model).export_onnx( + input_shape=(1, 3, 640, 640), output_dir=tmp_dir) + if __name__ == '__main__': unittest.main() From f82361d71b6c31a7607cee404ffc5a39c5d7e5a7 Mon Sep 17 00:00:00 2001 From: "feiyu.gfy" Date: Thu, 25 May 2023 18:28:05 +0800 Subject: [PATCH 51/76] =?UTF-8?q?=E4=BF=AE=E6=AD=A3lineless=5Ftable=5Freco?= =?UTF-8?q?gnition=E5=8A=9F=E8=83=BD=E9=81=87=E5=88=B0=E7=A9=BA=E7=99=BD?= =?UTF-8?q?=E5=9B=BE=E7=89=87=E5=B4=A9=E6=BA=83=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 用户反馈lineless_table_recognition功能遇到空白图片会发生崩溃。 增加处理机制以防止崩溃。 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12741201 * fix bug --- .../cv/table_recognition/lineless_table_process.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/modelscope/models/cv/table_recognition/lineless_table_process.py b/modelscope/models/cv/table_recognition/lineless_table_process.py index 0d7fcfb5..a04d531d 100644 --- a/modelscope/models/cv/table_recognition/lineless_table_process.py +++ b/modelscope/models/cv/table_recognition/lineless_table_process.py @@ -346,6 +346,7 @@ def merge_outputs(detections): def filter(results, logi, ps): # this function select boxes batch_size, feat_dim = logi.shape[0], logi.shape[2] + num_valid = sum(results[1][:, 8] >= 0.15) slct_logi = np.zeros((batch_size, num_valid, feat_dim), dtype=np.float32) @@ -358,6 +359,14 @@ def filter(results, logi, ps): return torch.Tensor(slct_logi).cuda(), torch.Tensor(slct_dets).cuda() +def normalized_ps(ps, vocab_size): + ps = torch.round(ps).to(torch.int64) + ps = torch.where(ps < vocab_size, ps, (vocab_size - 1) + * torch.ones(ps.shape).to(torch.int64).cuda()) + ps = torch.where(ps >= 0, ps, torch.zeros(ps.shape).to(torch.int64).cuda()) + return ps + + def process_detect_output(output, meta): K, MK = 3000, 5000 num_classes = 2 @@ -390,6 +399,7 @@ def process_detect_output(output, meta): logi = logi + cr results = merge_outputs(detections) slct_logi_feat, slct_dets_feat = filter(results, logi, raw_dets[:, :, :8]) + slct_dets_feat = normalized_ps(slct_dets_feat, 256) slct_output_dets = results[1][:slct_logi_feat.shape[1], :8] return slct_logi_feat, slct_dets_feat, slct_output_dets From c08b92496821c7fbc0b22dbd063d595b9ed024e2 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Thu, 25 May 2023 18:55:47 +0800 Subject: [PATCH 52/76] Fix ner tokenizer which won't accept kwargs Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12664244 * fix slow tokenizer --- .../nlp/token_classification_preprocessor.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/modelscope/preprocessors/nlp/token_classification_preprocessor.py b/modelscope/preprocessors/nlp/token_classification_preprocessor.py index 4b4fee1f..b3ff9935 100644 --- a/modelscope/preprocessors/nlp/token_classification_preprocessor.py +++ b/modelscope/preprocessors/nlp/token_classification_preprocessor.py @@ -388,10 +388,14 @@ class TokenClassificationTransformersPreprocessor( f'tokenizer {tokenizer_name}, please use a fast tokenizer instead, or ' f'try to implement a `{method}` method') label_mask, offset_mapping = getattr(self, method)(tokens) - padding = self.nlp_tokenizer.get_tokenizer_kwarg('padding') - max_length = self.nlp_tokenizer.get_tokenizer_kwarg('max_length') - special_token = 1 if self.nlp_tokenizer.get_tokenizer_kwarg( - 'add_special_tokens') else 0 + padding = kwargs.get('padding', + self.nlp_tokenizer.get_tokenizer_kwarg('padding')) + max_length = kwargs.get( + 'max_length', self.nlp_tokenizer.get_tokenizer_kwarg('max_length')) + special_token = 1 if kwargs.get( + 'add_special_tokens', + self.nlp_tokenizer.get_tokenizer_kwarg( + 'add_special_tokens')) else 0 if len(label_mask) > max_length - 2 * special_token: label_mask = label_mask[:(max_length - 2 * special_token)] offset_mapping = offset_mapping[:sum(label_mask)] From a7a3eb5dc57a084582e8b6362ff170a8c32cdbce Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Thu, 25 May 2023 19:39:24 +0800 Subject: [PATCH 53/76] Support flex train feature Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12737668 --- modelscope/hub/push_to_hub.py | 67 +++++++++++++++++++ modelscope/trainers/default_config.py | 2 +- .../hooks/checkpoint/checkpoint_hook.py | 59 +++++++++------- modelscope/trainers/trainer.py | 2 +- modelscope/trainers/training_args.py | 14 ++-- 5 files changed, 112 insertions(+), 32 deletions(-) diff --git a/modelscope/hub/push_to_hub.py b/modelscope/hub/push_to_hub.py index d117cc7f..2b2b4091 100644 --- a/modelscope/hub/push_to_hub.py +++ b/modelscope/hub/push_to_hub.py @@ -2,6 +2,8 @@ import concurrent.futures import os +import shutil +from multiprocessing import Manager, Process, Value from modelscope.hub.api import HubApi from modelscope.hub.constants import ModelVisibility @@ -11,6 +13,10 @@ from modelscope.utils.logger import get_logger logger = get_logger() _executor = concurrent.futures.ProcessPoolExecutor(max_workers=8) +_queues = dict() +_flags = dict() +_tasks = dict() +_manager = None def _api_push_to_hub(repo_name, @@ -131,3 +137,64 @@ def push_to_hub_async(repo_name, return _executor.submit(_api_push_to_hub, repo_name, output_dir, token, private, commit_message, tag, source_repo, ignore_file_pattern, revision) + + +def submit_task(q, b): + while True: + b.value = False + item = q.get() + logger.info(item) + b.value = True + if not item.pop('done', False): + delete_dir = item.pop('delete_dir', False) + output_dir = item.get('output_dir') + try: + push_to_hub(**item) + if delete_dir and os.path.exists(output_dir): + shutil.rmtree(output_dir) + except Exception as e: + logger.error(e) + else: + break + + +class UploadStrategy: + cancel = 'cancel' + wait = 'wait' + + +def push_to_hub_in_queue(queue_name, strategy=UploadStrategy.cancel, **kwargs): + assert queue_name is not None and len( + queue_name) > 0, 'Please specify a valid queue name!' + global _manager + if _manager is None: + _manager = Manager() + if queue_name not in _queues: + _queues[queue_name] = _manager.Queue() + _flags[queue_name] = Value('b', False) + process = Process( + target=submit_task, args=(_queues[queue_name], _flags[queue_name])) + process.start() + _tasks[queue_name] = process + + queue = _queues[queue_name] + flag: Value = _flags[queue_name] + if kwargs.get('done', False): + queue.put(kwargs) + elif flag.value and strategy == UploadStrategy.cancel: + logger.error( + f'Another uploading is running, ' + f'this uploading with message {kwargs.get("commit_message")} will be canceled.' + ) + else: + queue.put(kwargs) + + +def wait_for_done(queue_name): + process: Process = _tasks.pop(queue_name, None) + if process is None: + return + process.join() + + _queues.pop(queue_name) + _flags.pop(queue_name) diff --git a/modelscope/trainers/default_config.py b/modelscope/trainers/default_config.py index bb272695..e4b08b4a 100644 --- a/modelscope/trainers/default_config.py +++ b/modelscope/trainers/default_config.py @@ -57,7 +57,7 @@ def update_cfg(cfg: Config) -> Config: key_chain_map[_HOOK_KEY_CHAIN_MAP[key]] = value hook.clear() cfg.train.hooks = list(filter(bool, cfg.train.hooks)) - cfg.merge_from_dict(key_chain_map) + cfg.merge_from_dict(key_chain_map, force=False) return cfg diff --git a/modelscope/trainers/hooks/checkpoint/checkpoint_hook.py b/modelscope/trainers/hooks/checkpoint/checkpoint_hook.py index 4b14a13f..b8be0682 100644 --- a/modelscope/trainers/hooks/checkpoint/checkpoint_hook.py +++ b/modelscope/trainers/hooks/checkpoint/checkpoint_hook.py @@ -1,14 +1,16 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import os import random -import time +import shutil from typing import Optional +import json import numpy as np import torch from modelscope.hub.check_model import check_model_is_id -from modelscope.hub.push_to_hub import push_to_hub_async +from modelscope.hub.push_to_hub import (UploadStrategy, push_to_hub_in_queue, + wait_for_done) from modelscope.metainfo import Hooks from modelscope.trainers.hooks.builder import HOOKS from modelscope.trainers.hooks.checkpoint.checkpoint_processor import \ @@ -45,7 +47,9 @@ class CheckpointHook(Hook): hub_repo_id (str): The hub repo id. hub_token (str): The token of the modelhub. You can also set the environment variable `MODELSCOPE_API_TOKEN`. private_hub (bool): Whether push to a private hub, default True. - hub_revision (str): Which branch to push the model to, default is `master` + hub_revision (str): Which branch to push the model to, default is `master`. + upload_strategy (str): The action adopted when the previous uploading is not done + and the next one is coming, can be `cancel` or `wait`. kwargs: by_epoch (bool): Same with `save_strategy`, but has a higher priority, legacy argument. output_sub_dir (str): The folder under the `save_dir` to save the output checkpoint for inference. @@ -56,6 +60,8 @@ class CheckpointHook(Hook): EVAL_RESULT_FILE = 'eval_result.txt' + PUSH_TO_HUB_QUEUE_NAME = 'train.checkpoint' + def __init__(self, save_strategy: Optional[str] = CheckpointStrategy.by_epoch, interval: Optional[int] = 0, @@ -68,6 +74,7 @@ class CheckpointHook(Hook): hub_token: Optional[str] = None, private_hub: Optional[bool] = True, hub_revision: Optional[str] = DEFAULT_REPOSITORY_REVISION, + upload_strategy: Optional[str] = UploadStrategy.cancel, **kwargs): self.interval = interval self.save_dir = save_dir @@ -89,9 +96,9 @@ class CheckpointHook(Hook): self.hub_token = hub_token self.private_hub = private_hub self.hub_revision = hub_revision + self.upload_strategy = upload_strategy self.tag = -1 self.is_model_id = None - self.push_to_hub_future = None self.max_checkpoint_num = None if max_checkpoint_num is not None: self.max_checkpoint_num = max(int(max_checkpoint_num), 1) @@ -149,13 +156,15 @@ class CheckpointHook(Hook): f'Saving checkpoint at {trainer.iter + 1} iter') self._save_checkpoint(trainer, prefix) if is_master() and self.push_to_hub: - if self.push_to_hub_future is not None and not self.push_to_hub_future.done( - ): - self.logger.error( - f'Another uploading is running, ' - f'this uploading with message {prefix} will be canceled.') - return - self.push_to_hub_future = self._push_to_hub(trainer, prefix) + if self.upload_strategy == UploadStrategy.cancel: + output_dir = self.output_dir + delete_dir = False + else: + output_dir = self.output_dir + '_upload_' + prefix + shutil.copytree( + self.output_dir, output_dir, dirs_exist_ok=True) + delete_dir = True + self._push_to_hub(trainer, prefix, output_dir, delete_dir) def after_train_epoch(self, trainer): if self.save_strategy != CheckpointStrategy.by_epoch: @@ -172,32 +181,36 @@ class CheckpointHook(Hook): self._do_save(trainer, CheckpointStrategy.by_step) def after_run(self, trainer): - if self.push_to_hub_future is not None and not self.push_to_hub_future.done( - ): - self.logger.info('Train finished. Uploading models, waiting...') - while not self.push_to_hub_future.done(): - time.sleep(1) - self.logger.info('Uploading models done.') + self.logger.info('Train finished. Uploading models, waiting...') + push_to_hub_in_queue( + self.PUSH_TO_HUB_QUEUE_NAME, + strategy=self.upload_strategy, + done=True) + wait_for_done(self.PUSH_TO_HUB_QUEUE_NAME) + self.logger.info('Uploading models done.') - def _push_to_hub(self, trainer, prefix): + def _push_to_hub(self, trainer, prefix, output_dir, delete_dir=False): if self.is_model_id is None: self.is_model_id = check_model_is_id(trainer.input_model_id, self.hub_token) self.tag += 1 - return push_to_hub_async( - self.hub_repo_id, - self.output_dir, + return push_to_hub_in_queue( + self.PUSH_TO_HUB_QUEUE_NAME, + strategy=self.upload_strategy, + repo_name=self.hub_repo_id, + output_dir=output_dir, token=self.hub_token, private=self.private_hub, commit_message=prefix, tag=f'v1.{self.tag}', revision=self.hub_revision, - source_repo=trainer.input_model_id if self.is_model_id else '') + source_repo=trainer.input_model_id if self.is_model_id else '', + delete_dir=delete_dir) def save_evaluate_results(self, trainer): with open(os.path.join(self.output_dir, self.EVAL_RESULT_FILE), 'w') as f: - f.write(str(trainer.metric_values)) + f.write(json.dumps(trainer.metric_values)) def _save_checkpoint(self, trainer, prefix): """Save checkpoint files and remove obsolete ones diff --git a/modelscope/trainers/trainer.py b/modelscope/trainers/trainer.py index 278d6b44..c980de04 100644 --- a/modelscope/trainers/trainer.py +++ b/modelscope/trainers/trainer.py @@ -155,10 +155,10 @@ class EpochBasedTrainer(BaseTrainer): self.cfg_modify_fn = cfg_modify_fn # add default config merge_cfg(self.cfg) + self.cfg = self.rebuild_config(self.cfg) if 'cfg_options' in kwargs: self.cfg.merge_from_dict(kwargs['cfg_options']) self.cfg = update_cfg(self.cfg) - self.cfg = self.rebuild_config(self.cfg) if isinstance(model, (TorchModel, nn.Module)): self.model = model diff --git a/modelscope/trainers/training_args.py b/modelscope/trainers/training_args.py index 16453d7c..77b7743e 100644 --- a/modelscope/trainers/training_args.py +++ b/modelscope/trainers/training_args.py @@ -507,7 +507,7 @@ def build_dataset_from_file(filename): "text2": "sequence2", "label": "label", } - "split": 0.8, + "usage": 0.8, } ] """ @@ -541,16 +541,16 @@ def build_dataset_from_file(filename): lambda x: x, remove_columns=remove_columns, features=new_features).rename_columns(ds['column_mapping']) - split = ds['split'] - if isinstance(split, str): - assert split in ('train', 'val') - if split == 'train': + usage = ds['usage'] + if isinstance(usage, str): + assert usage in ('train', 'val') + if usage == 'train': train_set.append(dataset) else: eval_set.append(dataset) else: - assert isinstance(split, float) and 0 < split < 1 - ds_dict = dataset.train_test_split(train_size=split) + assert isinstance(usage, float) and 0 < usage < 1 + ds_dict = dataset.train_test_split(train_size=usage) train_set.append(ds_dict['train']) eval_set.append(ds_dict['test']) From e2a20fd8171e6eb74d732c998bfde2c51100dc7d Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Thu, 25 May 2023 20:05:04 +0800 Subject: [PATCH 54/76] fix merge_hook Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12718423 * fix merge_hook From 5b99bb24fed017806bf0ed853a155cff55d1c297 Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Thu, 25 May 2023 20:28:36 +0800 Subject: [PATCH 55/76] update version: 1.6.1rc0 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12754942 --- examples/pytorch/stable_diffusion/run_train.sh | 6 +++--- requirements/framework.txt | 2 +- setup.py | 4 ++-- tests/run_config.yaml | 18 +++++++++++++++--- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/examples/pytorch/stable_diffusion/run_train.sh b/examples/pytorch/stable_diffusion/run_train.sh index 0e551942..033c1636 100644 --- a/examples/pytorch/stable_diffusion/run_train.sh +++ b/examples/pytorch/stable_diffusion/run_train.sh @@ -2,11 +2,11 @@ PYTHONPATH=. torchrun examples/pytorch/stable_diffusion/finetune_stable_diffusio --model 'damo/multi-modal_efficient-diffusion-tuning-lora' \ --work_dir './tmp/stable_diffusion_tuning' \ --train_dataset_namespace 'damo' \ - --train_dataset_name 'controlnet_dataset_condition_fill50k' \ - --max_epochs 1 \ + --train_dataset_name 'buptwq/lora-stable-diffusion-finetune-dog' \ + --max_epochs 150 \ --save_ckpt_strategy 'by_epoch' \ --logging_interval 100 \ --train.dataloader.workers_per_gpu 0 \ --evaluation.dataloader.workers_per_gpu 0 \ - --train.optimizer.lr 1e-5 \ + --train.optimizer.lr 1e-4 \ --use_model_config true diff --git a/requirements/framework.txt b/requirements/framework.txt index 10a1ef63..e763ae63 100644 --- a/requirements/framework.txt +++ b/requirements/framework.txt @@ -5,7 +5,7 @@ einops filelock>=3.3.0 gast>=0.2.2 # for python3.7 python3.8 compatible -numpy<1.22.0 +numpy<=1.22.0 oss2 # for datasets compatible pandas<=1.5.3 diff --git a/setup.py b/setup.py index 7ebb8dae..98b12888 100644 --- a/setup.py +++ b/setup.py @@ -201,8 +201,8 @@ if __name__ == '__main__': 'ModelScope: bring the notion of Model-as-a-Service to life.', long_description=readme(), long_description_content_type='text/markdown', - author='Alibaba ModelScope team', - author_email='modelscope@list.alibaba-inc.com', + author='ModelScope team', + author_email='contact@modelscope.cn', keywords='python,nlp,science,cv,speech,multi-modal', url='https://github.com/modelscope/modelscope', packages=find_packages(exclude=('configs', 'demo')), diff --git a/tests/run_config.yaml b/tests/run_config.yaml index c3daf065..ba678468 100644 --- a/tests/run_config.yaml +++ b/tests/run_config.yaml @@ -67,7 +67,7 @@ isolated: # test cases that may require excessive anmount of GPU memory or run envs: default: # default env, case not in other env will in default, pytorch. dependencies: # requirement packages，pip install before test case run. - - numpy>=1.20 + - numpy>=1.20,<=1.21.0 - protobuf<4,>=3.20.2 tensorflow1x: # cases excuted tensorflow1.x framework. requirements: # requirements files run before test case run. @@ -87,9 +87,21 @@ envs: - test_image_style_transfer.py - test_image_portrait_stylization_trainer.py - test_language_identification.py + - test_language_guided_video_summarization_trainer.py + - test_motion_generation.py + - test_universal_matting.py + - test_dialog_modeling.py + - test_trainer.py + - test_abnormal_object_detection.py + - test_image_face_fusion.py - test_ocr_detection_db_trainer.py + - test_language_guided_video_summarization.py + - test_interactive_translation_pipeline.py + - test_image_defrcn_fewshot_trainer.py + - test_automatic_post_editing.py + - test_human_reconstruction.py + - test_nerf_recon_acc_trainer.py - test_nerf_recon_acc.py + - test_speech_signal_process.py - test_tensorboard_hook.py - test_efficient_diffusion_tuning_trainer.py - - test_trainer.py - - test_nerf_recon_acc_trainer.py From 003c33648a7a42c59aafa4040633b688e51b8773 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Mon, 29 May 2023 10:32:33 +0800 Subject: [PATCH 56/76] Fix CI: test merge dataset failed Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12771691 * fix CI * add teardown --- tests/trainers/cli/test_cli.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/trainers/cli/test_cli.py b/tests/trainers/cli/test_cli.py index b9fb7539..55a12b88 100644 --- a/tests/trainers/cli/test_cli.py +++ b/tests/trainers/cli/test_cli.py @@ -1,4 +1,5 @@ # Copyright (c) Alibaba, Inc. and its affiliates. +import os import unittest import json @@ -21,7 +22,7 @@ class TestCli(unittest.TestCase): 'sentence2': 'sentence2', 'label': 'label', }, - 'split': 0.8, + 'usage': 0.8, }, { 'dataset': { 'dataset_name': 'glue', @@ -33,11 +34,15 @@ class TestCli(unittest.TestCase): 'hypothesis': 'sentence2', 'label': 'label', }, - 'split': 'val', + 'usage': 'val', }] with open('./dataset.json', 'w') as f: json.dump(content, f) + def tearDown(self) -> None: + if os.path.exists('./dataset.json'): + os.remove('./dataset.json') + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_merge_dataset_from_file(self): dataset = MsDataset.load('clue', subset_name='cmnli', split='train') From c7aede97baf49111a596882ae554a12084d3cfc0 Mon Sep 17 00:00:00 2001 From: "zhangzhicheng.zzc" Date: Tue, 30 May 2023 11:16:39 +0800 Subject: [PATCH 57/76] add pattern to skip easycv.thirdparty Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12785287 * add pattern to remove * update pattern to remove * avoid import with build --- modelscope/utils/plugins.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/modelscope/utils/plugins.py b/modelscope/utils/plugins.py index 9d238e7d..e997f676 100644 --- a/modelscope/utils/plugins.py +++ b/modelscope/utils/plugins.py @@ -171,7 +171,21 @@ def import_plugins(plugins: List[str] = None) -> List[str]: for module_name in plugins: try: - import_module_and_submodules(module_name) + # TODO: include and exclude should be configurable, hard code now + import_module_and_submodules( + module_name, + include={ + 'easycv.toolkit.modelscope', + 'easycv.hooks', + 'easycv.models', + 'easycv.core', + 'easycv.toolkit', + 'easycv.predictors', + }, + exclude={ + 'easycv.toolkit.*', + 'easycv.*', + }) logger.info('Plugin %s available', module_name) imported_plugins.append(module_name) except ModuleNotFoundError as e: @@ -238,9 +252,10 @@ def import_module_and_submodules(package_name: str, path_string = '' if not path else path[0] # walk_packages only finds immediate children, so need to recurse. - for module_finder, name, _ in pkgutil.walk_packages(path): + for module_finder, name, _ in pkgutil.iter_modules(path): # Sometimes when you import third-party libraries that are on your path, # `pkgutil.walk_packages` returns those too, so we need to skip them. + # `pkgutil.iter_modules` avoid import those package if path_string and module_finder.path != path_string: # type: ignore[union-attr] continue if name.startswith('_'): @@ -250,7 +265,8 @@ def import_module_and_submodules(package_name: str, # skip tests continue subpackage = f'{package_name}.{name}' - import_module_and_submodules(subpackage, exclude=exclude) + import_module_and_submodules( + subpackage, include=include, exclude=exclude) except SystemExit as e: # this case is specific for easy_cv's tools/predict.py exit logger.warning( From 10c39b5ce1b0be5de2b0302b0939ee7ae5da41c5 Mon Sep 17 00:00:00 2001 From: "yuanzhi.zyz" Date: Wed, 31 May 2023 21:16:22 +0800 Subject: [PATCH 58/76] add new ocr recognition model (LightweightEdge) and some functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 增加了新轻量化端侧识别模型 LightweightEdge，并把原来CRNN和ConvNextViT的代码整理了 2. 增加batch inference支持 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12787905 --- modelscope/models/cv/ocr_recognition/model.py | 13 +- .../ocr_recognition/modules/CRNN/__init__.py | 0 .../modules/{crnn.py => CRNN/main_model.py} | 5 + .../modules/ConvNextViT/__init__.py | 0 .../modules/{ => ConvNextViT}/convnext.py | 0 .../main_model.py} | 6 +- .../modules/{ => ConvNextViT}/timm_tinyc.py | 0 .../modules/{ => ConvNextViT}/vitstr.py | 0 .../modules/LightweightEdge/__init__.py | 0 .../modules/LightweightEdge/main_model.py | 42 + .../LightweightEdge/nas_block/__init__.py | 1 + .../LightweightEdge/nas_block/layers.py | 790 ++++++++++++++++++ .../LightweightEdge/nas_block/mix_ops.py | 226 +++++ .../LightweightEdge/nas_block/proxyless.py | 138 +++ .../models/cv/ocr_recognition/preprocessor.py | 29 +- .../ocr_recognition_dataset.py | 2 +- .../pipelines/cv/ocr_recognition_pipeline.py | 2 +- tests/pipelines/test_ocr_recognition.py | 32 + 18 files changed, 1264 insertions(+), 22 deletions(-) create mode 100644 modelscope/models/cv/ocr_recognition/modules/CRNN/__init__.py rename modelscope/models/cv/ocr_recognition/modules/{crnn.py => CRNN/main_model.py} (92%) create mode 100644 modelscope/models/cv/ocr_recognition/modules/ConvNextViT/__init__.py rename modelscope/models/cv/ocr_recognition/modules/{ => ConvNextViT}/convnext.py (100%) rename modelscope/models/cv/ocr_recognition/modules/{convnextvit.py => ConvNextViT/main_model.py} (63%) rename modelscope/models/cv/ocr_recognition/modules/{ => ConvNextViT}/timm_tinyc.py (100%) rename modelscope/models/cv/ocr_recognition/modules/{ => ConvNextViT}/vitstr.py (100%) create mode 100644 modelscope/models/cv/ocr_recognition/modules/LightweightEdge/__init__.py create mode 100644 modelscope/models/cv/ocr_recognition/modules/LightweightEdge/main_model.py create mode 100644 modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/__init__.py create mode 100644 modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/layers.py create mode 100644 modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/mix_ops.py create mode 100644 modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/proxyless.py diff --git a/modelscope/models/cv/ocr_recognition/model.py b/modelscope/models/cv/ocr_recognition/model.py index 2406b6dc..3510de45 100644 --- a/modelscope/models/cv/ocr_recognition/model.py +++ b/modelscope/models/cv/ocr_recognition/model.py @@ -10,8 +10,9 @@ from modelscope.models.builder import MODELS from modelscope.utils.config import Config from modelscope.utils.constant import ModelFile, Tasks from modelscope.utils.logger import get_logger -from .modules.convnextvit import ConvNextViT -from .modules.crnn import CRNN +from .modules.ConvNextViT.main_model import ConvNextViT +from .modules.CRNN.main_model import CRNN +from .modules.LightweightEdge.main_model import LightweightEdge LOGGER = get_logger() @@ -85,6 +86,8 @@ class OCRRecognition(TorchModel): self.recognizer = ConvNextViT() elif cfgs.model.recognizer == 'CRNN': self.recognizer = CRNN() + elif cfgs.model.recognizer == 'LightweightEdge': + self.recognizer = LightweightEdge() else: raise TypeError( f'recognizer should be either ConvNextViT, CRNN, but got {cfgs.model.recognizer}' @@ -94,7 +97,7 @@ class OCRRecognition(TorchModel): model_dict = self.recognizer.state_dict() # remove prefix for finetuned models check_point = { - k.replace('recognizer.', ''): v + k.replace('recognizer.', '').replace('module.', ''): v for k, v in params_pretrained.items() } model_dict.update(check_point) @@ -134,9 +137,9 @@ class OCRRecognition(TorchModel): labels = batch['labels'] bs = inputs.shape[0] if self.do_chunking: - inputs = inputs.view(bs * 3, 1, self.target_height, 300) + inputs = inputs.view(bs * 3, 3, self.target_height, 300) else: - inputs = inputs.view(bs, 1, self.target_height, self.target_width) + inputs = inputs.view(bs, 3, self.target_height, self.target_width) output = self(inputs) probs = output['probs'].permute(1, 0, 2) _, label_length, label_flatten = self.encdec.encode(labels) diff --git a/modelscope/models/cv/ocr_recognition/modules/CRNN/__init__.py b/modelscope/models/cv/ocr_recognition/modules/CRNN/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modelscope/models/cv/ocr_recognition/modules/crnn.py b/modelscope/models/cv/ocr_recognition/modules/CRNN/main_model.py similarity index 92% rename from modelscope/models/cv/ocr_recognition/modules/crnn.py rename to modelscope/models/cv/ocr_recognition/modules/CRNN/main_model.py index 3de8304d..f2ce8c21 100644 --- a/modelscope/models/cv/ocr_recognition/modules/crnn.py +++ b/modelscope/models/cv/ocr_recognition/modules/CRNN/main_model.py @@ -79,6 +79,11 @@ class CRNN(nn.Module): self.cls = nn.Linear(512, 7644, bias=False) def forward(self, input): + # RGB2GRAY + input = input[:, 0: + 1, :, :] * 0.2989 + input[:, 1: + 2, :, :] * 0.5870 + input[:, 2: + 3, :, :] * 0.1140 feats = self.conv0(input) feats = self.p0(feats) feats = self.conv1(feats) diff --git a/modelscope/models/cv/ocr_recognition/modules/ConvNextViT/__init__.py b/modelscope/models/cv/ocr_recognition/modules/ConvNextViT/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modelscope/models/cv/ocr_recognition/modules/convnext.py b/modelscope/models/cv/ocr_recognition/modules/ConvNextViT/convnext.py similarity index 100% rename from modelscope/models/cv/ocr_recognition/modules/convnext.py rename to modelscope/models/cv/ocr_recognition/modules/ConvNextViT/convnext.py diff --git a/modelscope/models/cv/ocr_recognition/modules/convnextvit.py b/modelscope/models/cv/ocr_recognition/modules/ConvNextViT/main_model.py similarity index 63% rename from modelscope/models/cv/ocr_recognition/modules/convnextvit.py rename to modelscope/models/cv/ocr_recognition/modules/ConvNextViT/main_model.py index 4e7900b1..365d4005 100644 --- a/modelscope/models/cv/ocr_recognition/modules/convnextvit.py +++ b/modelscope/models/cv/ocr_recognition/modules/ConvNextViT/main_model.py @@ -14,7 +14,11 @@ class ConvNextViT(nn.Module): self.vitstr = vitstr_tiny(num_tokens=7644) def forward(self, input): - """ Transformation stage """ + # RGB2GRAY + input = input[:, 0: + 1, :, :] * 0.2989 + input[:, 1: + 2, :, :] * 0.5870 + input[:, 2: + 3, :, :] * 0.1140 features = self.cnn_model(input) output = self.vitstr(features) return output diff --git a/modelscope/models/cv/ocr_recognition/modules/timm_tinyc.py b/modelscope/models/cv/ocr_recognition/modules/ConvNextViT/timm_tinyc.py similarity index 100% rename from modelscope/models/cv/ocr_recognition/modules/timm_tinyc.py rename to modelscope/models/cv/ocr_recognition/modules/ConvNextViT/timm_tinyc.py diff --git a/modelscope/models/cv/ocr_recognition/modules/vitstr.py b/modelscope/models/cv/ocr_recognition/modules/ConvNextViT/vitstr.py similarity index 100% rename from modelscope/models/cv/ocr_recognition/modules/vitstr.py rename to modelscope/models/cv/ocr_recognition/modules/ConvNextViT/vitstr.py diff --git a/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/__init__.py b/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/main_model.py b/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/main_model.py new file mode 100644 index 00000000..08584b5c --- /dev/null +++ b/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/main_model.py @@ -0,0 +1,42 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from collections import OrderedDict + +import torch.nn as nn + +from .nas_block import plnas_linear_mix_se + + +class LightweightEdge(nn.Module): + """ + 基于混合rep block的nas模型 + Args: + input (tensor): batch of input images + """ + + def __init__(self): + super(LightweightEdge, self).__init__() + self.FeatureExtraction = plnas_linear_mix_se(3, 123) + self.AdaptiveAvgPool = nn.AdaptiveAvgPool2d( + (None, 1)) # Transform final (imgH/16-1) -> 1 + self.dropout = nn.Dropout(0.3) + self.Prediction = nn.Sequential( + OrderedDict([ + ('fc1', nn.Linear(123, 120)), + ('bn', nn.BatchNorm1d(120)), + ('fc2', nn.Linear(120, 7642)), + ])) + + def forward(self, input): + visual_feature = self.FeatureExtraction(input) + visual_feature = self.AdaptiveAvgPool( + visual_feature.permute(0, 3, 1, 2)) # [b, c, h, w] -> [b, w, c, h] + visual_feature = visual_feature.squeeze(3) + visual_feature = self.dropout(visual_feature) + prediction = self.Prediction.fc1(visual_feature.contiguous()) + b, t, c = prediction.shape + prediction = self.Prediction.bn(prediction.view(b * t, + c)).view(b, t, c) + prediction = self.Prediction.fc2(prediction) + + return prediction diff --git a/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/__init__.py b/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/__init__.py new file mode 100644 index 00000000..c8115b7c --- /dev/null +++ b/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/__init__.py @@ -0,0 +1 @@ +from .proxyless import plnas_linear_mix_se diff --git a/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/layers.py b/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/layers.py new file mode 100644 index 00000000..23802f6d --- /dev/null +++ b/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/layers.py @@ -0,0 +1,790 @@ +from collections import OrderedDict + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def get_same_padding(kernel_size): + + if isinstance(kernel_size, tuple): + assert len(kernel_size) == 2, 'invalid kernel size: %s' % kernel_size + p1 = get_same_padding(kernel_size[0]) + p2 = get_same_padding(kernel_size[1]) + return p1, p2 + assert isinstance(kernel_size, + int), 'kernel size should be either `int` or `tuple`' + assert kernel_size % 2 > 0, 'kernel size should be odd number' + return kernel_size // 2 + + +def count_conv_flop(layer, x): + out_h = int(x.size(2) / layer.stride[0]) + out_w = int(x.size(3) / layer.stride[1]) + delta_ops = layer.in_channels * layer.out_channels * layer.kernel_size[ + 0] * layer.kernel_size[1] * out_h * out_w / layer.groups + return delta_ops + + +def set_layer_from_config(layer_config): + if layer_config is None: + return None + name2layer = { + ZeroLayer.__name__: ZeroLayer, + MBInvertedConvLayer.__name__: MBInvertedConvLayer, + IdentityLayer.__name__: IdentityLayer, + } + layer_name = layer_config.pop('name') + layer = name2layer[layer_name] + return layer.build_from_config(layer_config) + + +class MobileInvertedResidualBlock(nn.Module): + + def __init__(self, mobile_inverted_conv, shortcut): + super(MobileInvertedResidualBlock, self).__init__() + self.mobile_inverted_conv = mobile_inverted_conv + self.shortcut = shortcut + + def forward(self, x): + if self.mobile_inverted_conv.is_zero_layer(): + res = x + elif self.shortcut is None or self.shortcut.is_zero_layer(): + res = self.mobile_inverted_conv(x) + else: + conv_x = self.mobile_inverted_conv(x) + skip_x = self.shortcut(x) + res = skip_x + conv_x + return res + + @property + def module_str(self): + return '(%s, %s)' % (self.mobile_inverted_conv.module_str, + self.shortcut.module_str + if self.shortcut is not None else None) + + @property + def config(self): + return { + 'name': + MobileInvertedResidualBlock.__name__, + 'mobile_inverted_conv': + self.mobile_inverted_conv.config, + 'shortcut': + self.shortcut.config if self.shortcut is not None else None, + } + + @staticmethod + def build_from_config(config): + mobile_inverted_conv = set_layer_from_config( + config['mobile_inverted_conv']) + shortcut = set_layer_from_config(config['shortcut']) + return MobileInvertedResidualBlock(mobile_inverted_conv, shortcut) + + def get_flops(self, x): + flops1, _ = self.mobile_inverted_conv.get_flops(x) + if self.shortcut: + flops2, _ = self.shortcut.get_flops(x) + else: + flops2 = 0 + return flops1 + flops2, self.forward(x) + + +class MBInvertedConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + stride=(1, 1), + expand_ratio=6, + mid_channels=None): + super(MBInvertedConvLayer, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.stride = stride + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + + feature_dim = round( + self.in_channels + * self.expand_ratio) if mid_channels is None else mid_channels + if self.expand_ratio == 1: + self.inverted_bottleneck = None + else: + self.inverted_bottleneck = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + self.in_channels, feature_dim, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + pad = get_same_padding(self.kernel_size) + self.depth_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + feature_dim, + feature_dim, + kernel_size, + stride, + pad, + groups=feature_dim, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + def forward(self, x): + if self.inverted_bottleneck: + x = self.inverted_bottleneck(x) + x = self.depth_conv(x) + x = self.point_conv(x) + return x + + @property + def module_str(self): + return '%dx%d_MBConv%d' % (self.kernel_size[0], self.kernel_size[1], + self.expand_ratio) + + @property + def config(self): + return { + 'name': MBInvertedConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'kernel_size': self.kernel_size, + 'stride': self.stride, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return MBInvertedConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + if self.inverted_bottleneck: + total_flops += count_conv_flop(self.inverted_bottleneck.conv, x) + x = self.inverted_bottleneck(x) + total_flops += count_conv_flop(self.depth_conv.conv, x) + x = self.depth_conv(x) + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x + + +class IdentityLayer(nn.Module): + + def __init__(self): + super(IdentityLayer, self).__init__() + + def forward(self, x): + return x + + @property + def module_str(self): + return 'Identity' + + @property + def config(self): + return { + 'name': IdentityLayer.__name__, + } + + @staticmethod + def build_from_config(config): + return IdentityLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + return 0, self.forward(x) + + +class ZeroLayer(nn.Module): + + def __init__(self, stride): + super(ZeroLayer, self).__init__() + self.stride = stride + + def forward(self, x): + n, c, h, w = x.shape + h //= self.stride[0] + w //= self.stride[1] + device = x.device + padding = torch.zeros(n, c, h, w, device=device, requires_grad=False) + return padding + + @property + def module_str(self): + return 'Zero' + + @property + def config(self): + return {'name': ZeroLayer.__name__, 'stride': self.stride} + + @staticmethod + def build_from_config(config): + return ZeroLayer(**config) + + @staticmethod + def is_zero_layer(): + return True + + def get_flops(self, x): + return 0, self.forward(x) + + +def split_layer(total_channels, num_groups): + split = [ + int(np.ceil(total_channels / num_groups)) for _ in range(num_groups) + ] + split[num_groups - 1] += total_channels - sum(split) + return split + + +class MBInvertedMixConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + mix_conv_size=[1, 3, 5], + stride=(1, 1), + expand_ratio=6, + mid_channels=None): + super(MBInvertedMixConvLayer, self).__init__() + self.mix_type = 0 + + self.in_channels = in_channels + self.out_channels = out_channels + self.mix_conv_size = mix_conv_size + self.stride = stride + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + + feature_dim = round( + self.in_channels + * self.expand_ratio) if mid_channels is None else mid_channels + self.inverted_bottleneck = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(self.in_channels, feature_dim, 1, 1, 0, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + + self.mix_conv_size = mix_conv_size + self.n_chunks = len(mix_conv_size) + self.split_in_channels = split_layer(feature_dim, self.n_chunks) + + self.mix_conv = nn.ModuleList() + for idx in range(self.n_chunks): + kernel_size = self.mix_conv_size[idx] + pad = get_same_padding(kernel_size) + split_in_channels_ = self.split_in_channels[idx] + if self.mix_type == 0: + self.mix_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + split_in_channels_, + split_in_channels_, + kernel_size, + stride, + pad, + groups=split_in_channels_, + bias=False)), + ('bn', nn.BatchNorm2d(split_in_channels_)), + ('act', nn.PReLU()), + ]))) + else: + self.mix_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + split_in_channels_, + split_in_channels_, + kernel_size, + stride, + pad, + groups=split_in_channels_, + bias=False)), + ('bn', nn.BatchNorm2d(split_in_channels_)), + ]))) + if self.mix_type != 0: + self.act = nn.PReLU() + + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + def forward(self, x): + x = self.inverted_bottleneck(x) + split = torch.split(x, self.split_in_channels, dim=1) + x = torch.cat([layer(s) for layer, s in zip(self.mix_conv, split)], + dim=1) + + if self.mix_type != 0: + x = self.act(x) + x = self.point_conv(x) + + return x + + @property + def module_str(self): + return '%s_MixConv%d' % (str(self.mix_conv_size), self.expand_ratio) + + @property + def config(self): + return { + 'name': MBInvertedMixConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'mix_conv_size': self.mix_conv_size, + 'stride': self.stride, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return MBInvertedMixConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + # if self.inverted_bottleneck: + total_flops += count_conv_flop(self.inverted_bottleneck.conv, x) + x = self.inverted_bottleneck(x) + split = torch.split(x, self.split_in_channels, dim=1) + out = [] + # import pdb;pdb.set_trace() + for layer, s in zip(self.mix_conv, split): + out.append(layer(s)) + total_flops += count_conv_flop(layer.conv, s) + x = torch.cat(out, dim=1) + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x + + +class LinearMixConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + mix_conv_size=[1, 3, 5], + stride=(1, 1), + mid_channels=None): + super(LinearMixConvLayer, self).__init__() + self.mix_type = 0 + + self.in_channels = in_channels + self.out_channels = out_channels + self.mix_conv_size = mix_conv_size + self.stride = stride + self.mid_channels = mid_channels + + self.mix_conv_size = mix_conv_size + self.n_chunks = len(mix_conv_size) + self.mix_conv = nn.ModuleList() + + for idx in range(self.n_chunks): + kernel_size = self.mix_conv_size[idx] + pad = get_same_padding(kernel_size) + if self.mix_type == 0: + self.mix_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + in_channels, + in_channels, + kernel_size, + stride, + pad, + groups=in_channels, + bias=False)), + ('bn', nn.BatchNorm2d(in_channels)), + ('act', nn.PReLU()), + ]))) + else: + self.mix_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + in_channels, + in_channels, + kernel_size, + stride, + pad, + groups=in_channels, + bias=False)), + ('bn', nn.BatchNorm2d(in_channels)), + ]))) + + if self.mix_conv != 0: + self.act = nn.PReLU() + + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + in_channels * self.n_chunks, + out_channels, + 1, + 1, + 0, + bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + def forward(self, x): + x = torch.cat([layer(x) for layer in self.mix_conv], dim=1) + if self.mix_conv != 0: + x = self.act(x) + x = self.point_conv(x) + return x + + @property + def module_str(self): + return '%s_LinearMixConv' % (str(self.mix_conv_size)) + + @property + def config(self): + return { + 'name': LinearMixConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'mix_conv_size': self.mix_conv_size, + 'stride': self.stride, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return LinearMixConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + out = [] + for layer in self.mix_conv: + out.append(layer(x)) + total_flops += count_conv_flop(layer.conv, x) + x = torch.cat(out, dim=1) + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x + + +class SELayer(nn.Module): + ''' + ''' + + def __init__(self, input_channels: int, squeeze_factor: int = 4): + super(SELayer, self).__init__() + self.input_channels = input_channels + self.squeeze_factor = squeeze_factor + self.squeeze_channels = input_channels // squeeze_factor + self.fc1 = nn.Conv2d( + self.input_channels, self.squeeze_channels, 1, bias=True) + self.relu = nn.ReLU(inplace=True) + self.fc2 = nn.Conv2d( + self.squeeze_channels, self.input_channels, 1, bias=True) + + def _scale(self, input): + scale = F.adaptive_avg_pool2d(input, 1) + scale = self.fc1(scale) + scale = self.relu(scale) + scale = self.fc2(scale) + return torch.sigmoid(scale) + + def forward(self, input): + scale = self._scale(input) + return scale * input + + @property + def module_str(self): + return 'SE_%d' % (self.squeeze_factor) + + @property + def config(self): + return { + 'name': SELayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'squeeze_factor': self.squeeze_factor, + } + + @staticmethod + def build_from_config(config): + return SELayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + ''' + count se flops, only compute the fc layers' calculation + ''' + total_flops = 0 + total_flops += self.input_channels * self.squeeze_channels * 2 + b, c, h, w = x.shape + total_flops += c * h * w + return total_flops, x + + +class MBInvertedRepConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + rep_conv_size=[1, 3, 5], + stride=(1, 1), + expand_ratio=6, + mid_channels=None): + super(MBInvertedRepConvLayer, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.rep_conv_size = rep_conv_size + self.stride = stride + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + + feature_dim = round( + self.in_channels + * self.expand_ratio) if mid_channels is None else mid_channels + + self.inverted_bottleneck = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(self.in_channels, feature_dim, 1, 1, 0, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + + self.rep_conv_size = rep_conv_size + self.n_chunks = len(rep_conv_size) + + self.rep_conv = nn.ModuleList() + for idx in range(self.n_chunks): + kernel_size = self.rep_conv_size[idx] + pad = get_same_padding(kernel_size) + self.rep_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + feature_dim, + feature_dim, + kernel_size, + stride, + pad, + groups=feature_dim, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ]))) + + self.rep_conv_deploy = None + self.act = nn.PReLU() + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + self.deploy = False + + def forward(self, x): + # if self.inverted_bottleneck: + + x = self.inverted_bottleneck(x) + + if not self.deploy: + out = [] + for layer in self.rep_conv: + out.append(layer(x)) + x = out[0] + for out_ in out[1:]: + x += out_ + else: + x = self.rep_conv_deploy(x) + x = self.act(x) + x = self.point_conv(x) + return x + + @property + def module_str(self): + return '%s_RepConv%d' % (str(self.rep_conv_size), self.expand_ratio) + + @property + def config(self): + return { + 'name': MBInvertedMixConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'rep_conv_size': self.rep_conv_size, + 'stride': self.stride, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return MBInvertedMixConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def switch_to_deploy(self): + self.deploy = True + feature_dim = self.rep_conv[0].conv.in_channels + stride = self.rep_conv[0].conv.stride + kernel_size = max(self.rep_conv_size) + pad = get_same_padding(kernel_size) + self.rep_conv_deploy = nn.Conv2d( + feature_dim, + feature_dim, + kernel_size, + stride, + pad, + groups=feature_dim, + bias=True) + + kernel, bias = self.get_equivalent_kernel_bias() + self.rep_conv_deploy.weight.data = kernel + self.rep_conv_deploy.bias.data = bias + for para in self.parameters(): + para.detach_() + self.__delattr__('rep_conv') + + def get_equivalent_kernel_bias(self): + import pdb + pdb.set_trace() + if 1 in self.rep_conv_size: + kernel1x1, bias1x1 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(1)]) + else: + kernel1x1 = None + bias1x1 = 0 + + if 3 in self.rep_conv_size: + kernel3x3, bias3x3 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(3)]) + else: + kernel3x3 = None + bias3x3 = 0 + + if 5 in self.rep_conv_size: + kernel5x5, bias5x5 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(5)]) + + else: + kernel5x5 = 0 + bias5x5 = 0 + + return kernel5x5 + self._pad_1x1_to_5x5_tensor( + kernel1x1) + self._pad_3x3_to_5x5_tensor( + kernel3x3), bias5x5 + bias3x3 + bias1x1 + + def _pad_1x1_to_5x5_tensor(self, kernel1x1): + if kernel1x1 is None: + return 0 + else: + return torch.nn.functional.pad(kernel1x1, [2, 2, 2, 2]) + + def _pad_3x3_to_5x5_tensor(self, kernel3x3): + if kernel3x3 is None: + return 0 + else: + return torch.nn.functional.pad(kernel3x3, [1, 1, 1, 1]) + + def _fuse_bn_tensor(self, branch): + if branch is None: + return 0, 0 + if isinstance(branch, nn.Sequential): + kernel = branch.conv.weight + running_mean = branch.bn.running_mean + running_var = branch.bn.running_var + gamma = branch.bn.weight + beta = branch.bn.bias + eps = branch.bn.eps + else: + assert isinstance(branch, nn.BatchNorm2d) + if not hasattr(self, 'id_tensor'): + input_dim = self.in_channels // self.groups + kernel_value = np.zeros((self.in_channels, input_dim, 3, 3), + dtype=np.float32) + for i in range(self.in_channels): + kernel_value[i, i % input_dim, 1, 1] = 1 + self.id_tensor = torch.from_numpy(kernel_value).to( + branch.weight.device) + kernel = self.id_tensor + running_mean = branch.running_mean + running_var = branch.running_var + gamma = branch.weight + beta = branch.bias + eps = branch.eps + std = (running_var + eps).sqrt() + t = (gamma / std).reshape(-1, 1, 1, 1) + + return kernel * t, beta - running_mean * gamma / std + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + total_flops += count_conv_flop(self.inverted_bottleneck.conv, x) + x = self.inverted_bottleneck(x) + + total_flops += count_conv_flop(self.rep_conv[-1].conv, x) + out = [] + for layer in self.rep_conv: + out.append(layer(x)) + x = out[0] + for out_ in out[1:]: + x += out_ + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x diff --git a/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/mix_ops.py b/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/mix_ops.py new file mode 100644 index 00000000..effed1a6 --- /dev/null +++ b/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/mix_ops.py @@ -0,0 +1,226 @@ +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.parameter import Parameter + +from .layers import (LinearMixConvLayer, MBInvertedConvLayer, + MBInvertedMixConvLayer, MBInvertedRepConvLayer, SELayer, + ZeroLayer) + + +def detach_variable(inputs): + if isinstance(inputs, tuple): + return tuple([detach_variable(x) for x in inputs]) + else: + x = inputs.detach() + x.requires_grad = inputs.requires_grad + return x + + +def delta_ij(i, j): + if i == j: + return 1 + else: + return 0 + + +def conv_func_by_name(name): + + name2ops = { + 'Identity': + lambda in_C, out_C, S, height: IdentityLayer( + in_C, out_C, ops_order=ops_order), + 'Zero': + lambda in_C, out_C, S, height: ZeroLayer(stride=S), + } + name2ops.update({ + '3x3_MBConv1': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 3), 3), S, 1), + '3x3_MBConv2': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 3), 3), S, 2), + '3x3_MBConv3': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 3), 3), S, 3), + '3x3_MBConv4': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 3), 3), S, 4), + '3x3_MBConv5': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 3), 3), S, 5), + '3x3_MBConv6': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 3), 3), S, 6), + ####################################################################################### + '5x5_MBConv1': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 5), 5), S, 1), + '5x5_MBConv2': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 5), 5), S, 2), + '5x5_MBConv3': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 5), 5), S, 3), + '5x5_MBConv4': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 5), 5), S, 4), + '5x5_MBConv5': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 5), 5), S, 5), + '5x5_MBConv6': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 5), 5), S, 6), + ####################################################################################### + '7x7_MBConv1': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 7), 7), S, 1), + '7x7_MBConv2': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 7), 7), S, 2), + '7x7_MBConv3': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 7), 7), S, 3), + '7x7_MBConv4': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 7), 7), S, 4), + '7x7_MBConv5': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 7), 7), S, 5), + '7x7_MBConv6': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 7), 7), S, 6), + ####################################################################################### + '13_MixConv1': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 1), + '13_MixConv2': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 2), + '13_MixConv3': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 3), + '13_MixConv4': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 4), + '13_MixConv5': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 5), + '13_MixConv6': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 6), + ####################################################################################### + '35_MixConv1': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 1), + '35_MixConv2': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 2), + '35_MixConv3': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 3), + '35_MixConv4': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 4), + '35_MixConv5': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 5), + '35_MixConv6': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 6), + ####################################################################################### + '135_MixConv1': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 1), + '135_MixConv2': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 2), + '135_MixConv3': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 3), + '135_MixConv4': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 4), + '135_MixConv5': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 5), + '135_MixConv6': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 6), + ####################################################################################### + '13_LinMixConv': + lambda in_C, out_C, S, height: LinearMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S), + '35_LinMixConv': + lambda in_C, out_C, S, height: LinearMixConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S), + '135_LinMixConv': + lambda in_C, out_C, S, height: LinearMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S), + ####################################################################################### + 'SE_2': + lambda in_C, out_C, S, height: SELayer(in_C, 2), + 'SE_4': + lambda in_C, out_C, S, height: SELayer(in_C, 4), + 'SE_8': + lambda in_C, out_C, S, height: SELayer(in_C, 8), + ####################################################################################### + '13_RepConv1': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 1), + '13_RepConv2': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 2), + '13_RepConv3': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 3), + '13_RepConv4': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 4), + '13_RepConv5': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 5), + '13_RepConv6': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 6), + ####################################################################################### + '35_RepConv1': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 1), + '35_RepConv2': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 2), + '35_RepConv3': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 3), + '35_RepConv4': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 4), + '35_RepConv5': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 5), + '35_RepConv6': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 6), + ####################################################################################### + '135_RepConv1': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 1), + '135_RepConv2': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 2), + '135_RepConv3': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 3), + '135_RepConv4': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 4), + '135_RepConv5': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 5), + '135_RepConv6': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 6), + }) + return name2ops[name] diff --git a/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/proxyless.py b/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/proxyless.py new file mode 100644 index 00000000..4f525639 --- /dev/null +++ b/modelscope/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/proxyless.py @@ -0,0 +1,138 @@ +# Part of the implementation is borrowed and modified from ProxylessNAS, +# publicly available at https://github.com/mit-han-lab/proxylessnas +# paper linking at https://arxiv.org/pdf/1812.00332.pdf +import re +import sys +from queue import Queue + +import numpy as np +import torch +import torch.nn as nn + +from .layers import IdentityLayer, MobileInvertedResidualBlock +from .mix_ops import conv_func_by_name + + +class NasRecBackbone(nn.Module): + + def __init__(self, first_conv, blocks): + super(NasRecBackbone, self).__init__() + self.first_conv = first_conv + self.blocks = nn.ModuleList(blocks) + + def forward(self, x): + x = self.first_conv(x) + for block in self.blocks: + x = block(x) + return x + + def get_flops(self, x): + expected_flops = 0 + # first conv + flop = count_conv_flop(self.first_conv[0], x) + x = self.first_conv(x) + expected_flops += flop + # blocks + for mb_conv in self.blocks: + assert isinstance(mb_conv, MobileInvertedResidualBlock) + if mb_conv.shortcut is None: + shortcut_flop = 0 + else: + shortcut_flop, _ = mb_conv.shortcut.get_flops(x) + expected_flops += shortcut_flop + expected_flops += mb_conv.get_flops(x)[0] + + x = mb_conv(x) + return expected_flops + + +class CompactRecBackboneMixSE(NasRecBackbone): + + def __init__(self, first_stride, input_channel, stride_stages, + n_cell_stages, width_stages, conv_op_ids, conv_candidates, + se_candidates): + input_block_channel = 24 + first_conv = nn.Sequential( + nn.Conv2d( + input_channel, + input_block_channel, + kernel_size=(3, 3), + stride=first_stride, + padding=1, + bias=False), nn.BatchNorm2d(input_block_channel), nn.PReLU()) + + assert len(conv_op_ids) - 4 == sum(n_cell_stages) + blocks = [] + img_height = 16 + height_flag = 0 + for width, n_cell, s in zip(width_stages, n_cell_stages, + stride_stages): + for i in range(n_cell): + if i == 1: + img_height = int(img_height / 2) + + if img_height % 2 == 0: + height_flag = 1 + else: + height_flag = 0 + + if i == 0: + stride = s + else: + stride = (1, 1) + block_i = len(blocks) + conv_op = conv_func_by_name( + conv_candidates[conv_op_ids[block_i]])( + input_block_channel, width, stride, + img_height + height_flag) + if stride == (1, 1) and input_block_channel == width: + shortcut = IdentityLayer() + else: + shortcut = None + inverted_residual_block = MobileInvertedResidualBlock( + conv_op, shortcut) + blocks.append(inverted_residual_block) + input_block_channel = width + + block_i = len(blocks) + se_op = conv_func_by_name(se_candidates[conv_op_ids[block_i]])( + input_block_channel, width, stride, img_height) + + inverted_residual_block = MobileInvertedResidualBlock(se_op, None) + blocks.append(inverted_residual_block) + self.out_channel = input_block_channel + + super(CompactRecBackboneMixSE, self).__init__(first_conv, blocks) + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + +def plnas_linear_mix_se(input_channel, output_channel): + conv_candidates = [ + '5x5_MBConv2', '5x5_MBConv4', '5x5_MBConv6', '3x3_MBConv2', + '3x3_MBConv4', '3x3_MBConv6', '13_MixConv2', '13_MixConv4', + '13_MixConv6', '35_MixConv2', '35_MixConv4', '35_MixConv6', + '135_MixConv2', '135_MixConv4', '135_MixConv6', '13_LinMixConv', + '35_LinMixConv', '135_LinMixConv', '13_RepConv2', '13_RepConv4', + '13_RepConv6', '35_RepConv2', '35_RepConv4', '35_RepConv6', + '135_RepConv2', '135_RepConv4', '135_RepConv6', 'Zero' + ] + se_candidates = ['SE_2', 'SE_4', 'SE_8', 'Zero'] + + stride_stages = [(2, 2), (2, 1), (2, 1), (2, 1)] + n_cell_stages = [5, 5, 5, 5] + width_stages = [32, 64, 96, 123] + conv_op_ids = [ + 2, 23, 24, 26, 2, 2, 11, 27, 27, 27, 27, 2, 0, 2, 16, 10, 27, 2, 2, 2, + 22, 10, 27, 3 + ] + net = CompactRecBackboneMixSE(2, input_channel, stride_stages, + n_cell_stages, width_stages, conv_op_ids, + conv_candidates, se_candidates) + + return net diff --git a/modelscope/models/cv/ocr_recognition/preprocessor.py b/modelscope/models/cv/ocr_recognition/preprocessor.py index d327cd3c..b111637f 100644 --- a/modelscope/models/cv/ocr_recognition/preprocessor.py +++ b/modelscope/models/cv/ocr_recognition/preprocessor.py @@ -31,8 +31,6 @@ class OCRRecognitionPreprocessor(Preprocessor): self.do_chunking = cfgs.model.inference_kwargs.do_chunking self.target_height = cfgs.model.inference_kwargs.img_height self.target_width = cfgs.model.inference_kwargs.img_width - if self.do_chunking: - self.target_width = 804 def keepratio_resize(self, img): cur_ratio = img.shape[1] / float(img.shape[0]) @@ -45,8 +43,8 @@ class OCRRecognitionPreprocessor(Preprocessor): cur_target_height = self.target_height cur_target_width = int(self.target_height * cur_ratio) img = cv2.resize(img, (cur_target_width, cur_target_height)) - mask = np.zeros([mask_height, mask_width]).astype(np.uint8) - mask[:img.shape[0], :img.shape[1]] = img + mask = np.zeros([mask_height, mask_width, 3]).astype(np.uint8) + mask[:img.shape[0], :img.shape[1], :] = img img = mask return img @@ -65,29 +63,32 @@ class OCRRecognitionPreprocessor(Preprocessor): data_batch = [] for item in inputs: if isinstance(item, str): - img = np.array(load_image(item).convert('L')) + img = np.array(load_image(item).convert('RGB')) elif isinstance(item, PIL.Image.Image): - img = np.array(item.convert('L')) + img = np.array(item.convert('RGB')) elif isinstance(item, np.ndarray): - if len(item.shape) == 3: - img = cv2.cvtColor(item, cv2.COLOR_RGB2GRAY) + if len(item.shape) == 2: + img = cv2.cvtColor(item, cv2.COLOR_GRAY2RGB) + else: + img = item else: raise TypeError( f'inputs should be either (a list of) str, PIL.Image, np.array, but got {type(item)}' ) - img = self.keepratio_resize(img) img = torch.FloatTensor(img) if self.do_chunking: chunk_img = [] for i in range(3): left = (300 - 48) * i - chunk_img.append(img[:, left:left + 300]) + chunk_img.append(img[:, left:left + 300, :]) merge_img = torch.cat(chunk_img, 0) - data = merge_img.view(3, 1, self.target_height, 300) / 255. + data = merge_img.view(3, self.target_height, 300, 3) / 255. + data = data.permute(0, 3, 1, 2) else: - data = img.view(1, 1, self.target_height, - self.target_width) / 255. + data = img.view(1, self.target_height, self.target_width, + 3) / 255. + data = data.permute(0, 3, 1, 2) data_batch.append(data) data_batch = torch.cat(data_batch, 0) - return data_batch + return {'image': data_batch} diff --git a/modelscope/msdatasets/dataset_cls/custom_datasets/ocr_recognition_dataset.py b/modelscope/msdatasets/dataset_cls/custom_datasets/ocr_recognition_dataset.py index bfbb6eb3..07c6cdcc 100644 --- a/modelscope/msdatasets/dataset_cls/custom_datasets/ocr_recognition_dataset.py +++ b/modelscope/msdatasets/dataset_cls/custom_datasets/ocr_recognition_dataset.py @@ -68,7 +68,7 @@ class OCRRecognitionDataset(TorchCustomDataset): buf.seek(0) img = Image.open(buf).convert('L') if self.reco_preprocess is not None: - img = self.reco_preprocess(img) + img = self.reco_preprocess(img)['image'] label_key = 'label-%09d' % index label = txn.get(label_key.encode()).decode('utf-8') diff --git a/modelscope/pipelines/cv/ocr_recognition_pipeline.py b/modelscope/pipelines/cv/ocr_recognition_pipeline.py index e81e1ff6..602e23d4 100644 --- a/modelscope/pipelines/cv/ocr_recognition_pipeline.py +++ b/modelscope/pipelines/cv/ocr_recognition_pipeline.py @@ -66,7 +66,7 @@ class OCRRecognitionPipeline(Pipeline): return outputs def forward(self, inputs): - outputs = self.ocr_recognizer(inputs) + outputs = self.ocr_recognizer(inputs['image']) return outputs def postprocess(self, inputs): diff --git a/tests/pipelines/test_ocr_recognition.py b/tests/pipelines/test_ocr_recognition.py index 94ee521f..27870b10 100644 --- a/tests/pipelines/test_ocr_recognition.py +++ b/tests/pipelines/test_ocr_recognition.py @@ -20,6 +20,21 @@ class OCRRecognitionTest(unittest.TestCase): result = pipeline(input_location) print('ocr recognition results: ', result) + def pipeline_inference_batch(self, pipeline: Pipeline, + input_location: str): + result = pipeline( + [input_location, input_location, input_location, input_location], + batch_size=4) + print('ocr recognition results: ', result) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_from_modelhub_batch(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, + model=self.model_id, + model_revision='v2.3.0') + self.pipeline_inference_batch(ocr_recognition, self.test_image) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_run_with_model_from_modelhub(self): ocr_recognition = pipeline( @@ -68,6 +83,14 @@ class OCRRecognitionTest(unittest.TestCase): model_revision='v2.2.2') self.pipeline_inference(ocr_recognition, self.test_image) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_from_modelhub_lightweightedge(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, + model='damo/cv_LightweightEdge_ocr-recognitoin-general_damo', + model_revision='v1.0.0') + self.pipeline_inference(ocr_recognition, self.test_image) + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_run_with_model_from_modelhub_PILinput(self): ocr_recognition = pipeline( @@ -137,6 +160,15 @@ class OCRRecognitionTest(unittest.TestCase): device='cpu') self.pipeline_inference(ocr_recognition, self.test_image) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_from_modelhub_lightweightedge_cpu(self): + ocr_recognition = pipeline( + Tasks.ocr_recognition, + model='damo/cv_LightweightEdge_ocr-recognitoin-general_damo', + model_revision='v1.0.0', + device='cpu') + self.pipeline_inference(ocr_recognition, self.test_image) + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_run_with_model_from_modelhub_PILinput_cpu(self): ocr_recognition = pipeline( From 898e3a42eb7714656dea0b0898468ccae6a97e26 Mon Sep 17 00:00:00 2001 From: "hemu.zp" Date: Wed, 31 May 2023 21:29:36 +0800 Subject: [PATCH 59/76] Fix bug for DeepspeedHook.register_processor Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12686921 --- modelscope/trainers/hooks/distributed/deepspeed_hook.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modelscope/trainers/hooks/distributed/deepspeed_hook.py b/modelscope/trainers/hooks/distributed/deepspeed_hook.py index 7dddc5d9..d0a6eb9b 100644 --- a/modelscope/trainers/hooks/distributed/deepspeed_hook.py +++ b/modelscope/trainers/hooks/distributed/deepspeed_hook.py @@ -17,7 +17,7 @@ from modelscope.trainers.hooks.priority import Priority from modelscope.utils.checkpoint import save_checkpoint from modelscope.utils.logger import get_logger from ..checkpoint.checkpoint_processor import CheckpointProcessor -from ..lr_scheduler_hook import LrSchedulerProcessor +from ..lr_scheduler_hook import LrSchedulerHook, LrSchedulerProcessor from ..optimizer.base import OptimizerHook, OptimizerProcessor @@ -158,6 +158,10 @@ class DeepspeedHook(Hook): if len(optimizer_hook) > 0 and not isinstance( optimizer_hook[0].processor, DeepspeedProcessor): optimizer_hook[0].set_processor(processor) + lr_schedular_hook = trainer.get_hook(LrSchedulerHook) + if len(lr_schedular_hook) > 0 and not isinstance( + lr_schedular_hook[0].processor, DeepspeedProcessor): + lr_schedular_hook[0].set_processor(processor) ckpt_hook = trainer.get_hook(CheckpointHook) if len(ckpt_hook) > 0 and not isinstance(ckpt_hook[0].processor, DeepspeedProcessor): From 1b7e0f50f4cdeec45fb34b6facd23ab3dc774b0e Mon Sep 17 00:00:00 2001 From: "xixing.tj" Date: Wed, 31 May 2023 21:32:46 +0800 Subject: [PATCH 60/76] add ocr detection new model db-nas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增5M的DB-NAS ocr detection 文字检测模型 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12760623 * add ocr detection new model db-nas * add comment --- modelscope/models/cv/ocr_detection/model.py | 4 +- .../models/cv/ocr_detection/modules/dbnet.py | 241 +++++ .../models/cv/ocr_detection/modules/layers.py | 906 ++++++++++++++++++ .../cv/ocr_detection/modules/mix_ops.py | 689 +++++++++++++ .../cv/ocr_detection/modules/proxyless.py | 178 ++++ tests/pipelines/test_ocr_detection.py | 10 + 6 files changed, 2027 insertions(+), 1 deletion(-) create mode 100644 modelscope/models/cv/ocr_detection/modules/layers.py create mode 100644 modelscope/models/cv/ocr_detection/modules/mix_ops.py create mode 100644 modelscope/models/cv/ocr_detection/modules/proxyless.py diff --git a/modelscope/models/cv/ocr_detection/model.py b/modelscope/models/cv/ocr_detection/model.py index 712973ce..5e1728bf 100644 --- a/modelscope/models/cv/ocr_detection/model.py +++ b/modelscope/models/cv/ocr_detection/model.py @@ -12,7 +12,7 @@ from modelscope.models.builder import MODELS from modelscope.utils.config import Config from modelscope.utils.constant import ModelFile, Tasks from modelscope.utils.logger import get_logger -from .modules.dbnet import DBModel, VLPTModel +from .modules.dbnet import DBModel, DBNasModel, VLPTModel from .utils import boxes_from_bitmap, polygons_from_bitmap LOGGER = get_logger() @@ -40,6 +40,8 @@ class OCRDetection(TorchModel): self.detector = VLPTModel() elif self.backbone == 'resnet18': self.detector = DBModel() + elif self.backbone == 'proxylessnas': + self.detector = DBNasModel() else: raise TypeError( f'detector backbone should be either resnet18, resnet50, but got {cfgs.model.backbone}' diff --git a/modelscope/models/cv/ocr_detection/modules/dbnet.py b/modelscope/models/cv/ocr_detection/modules/dbnet.py index 82b0e512..2745dbf4 100644 --- a/modelscope/models/cv/ocr_detection/modules/dbnet.py +++ b/modelscope/models/cv/ocr_detection/modules/dbnet.py @@ -10,6 +10,8 @@ from collections import OrderedDict import torch import torch.nn as nn +from .proxyless import CompactDetBackbone + BatchNorm2d = nn.BatchNorm2d @@ -30,6 +32,73 @@ def conv3x3(in_planes, out_planes, stride=1): bias=False) +class DwPwConv(nn.Module): + + def __init__(self, + in_planes, + out_planes, + kernel_size, + stride=1, + padding=1, + bias=False): + super(DwPwConv, self).__init__() + self.depthwise = nn.Conv2d( + in_planes, + in_planes, + kernel_size, + stride, + padding, + groups=in_planes, + bias=bias) + self.bn1 = nn.BatchNorm2d(in_planes) + self.relu1 = nn.ReLU(inplace=True) + + self.pointwise = nn.Conv2d( + in_planes, + out_planes, + kernel_size=1, + stride=1, + padding=0, + groups=1, + bias=bias) + + def forward(self, x): + out = self.depthwise(x) + out = self.bn1(out) + out = self.relu1(out) + + out = self.pointwise(out) + + return out + + +class DwPwConvTranspose(nn.Module): + + def __init__(self, in_planes, out_planes, kernel_size, stride): + super(DwPwConvTranspose, self).__init__() + self.depthwise = nn.ConvTranspose2d( + in_planes, in_planes, kernel_size, stride, groups=in_planes) + self.bn1 = nn.BatchNorm2d(in_planes) + self.relu1 = nn.ReLU(inplace=True) + + self.pointwise = nn.Conv2d( + in_planes, + out_planes, + kernel_size=1, + stride=1, + padding=0, + groups=1) + + def forward(self, x): + out = self.depthwise(x) + out = self.bn1(out) + out = self.relu1(out) + + out = self.pointwise(out) + + return out + + class BasicBlock(nn.Module): expansion = 1 @@ -266,6 +335,156 @@ class ResNet(nn.Module): return x2, x3, x4, x5 +class LightSegDetector(nn.Module): + + def __init__(self, + in_channels=[64, 128, 256, 512], + inner_channels=256, + k=10, + bias=False, + adaptive=False, + smooth=False, + serial=False, + dw_kernel_size=3, + dw_padding=1, + *args, + **kwargs): + ''' + bias: Whether conv layers have bias or not. + adaptive: Whether to use adaptive threshold training or not. + smooth: If true, use bilinear instead of deconv. + serial: If true, thresh prediction will combine segmentation result as input. + ''' + super(LightSegDetector, self).__init__() + self.k = k + self.serial = serial + self.inner_channels = inner_channels + self.bias = bias + self.dw_kernel_size = dw_kernel_size + self.dw_padding = dw_padding + + self.up5 = nn.Upsample(scale_factor=8, mode='nearest') + self.up4 = nn.Upsample(scale_factor=4, mode='nearest') + self.up3 = nn.Upsample(scale_factor=2, mode='nearest') + + self.in5 = nn.Conv2d(in_channels[-1], inner_channels, 1, bias=bias) + self.in4 = nn.Conv2d(in_channels[-2], inner_channels, 1, bias=bias) + self.in3 = nn.Conv2d(in_channels[-3], inner_channels, 1, bias=bias) + self.in2 = nn.Conv2d(in_channels[-4], inner_channels, 1, bias=bias) + + # DwPM + self.binarize = nn.Sequential( + DwPwConv( + inner_channels, + inner_channels // 4, + kernel_size=self.dw_kernel_size, + padding=self.dw_padding, + bias=bias), BatchNorm2d(inner_channels // 4), + nn.ReLU(inplace=True), + DwPwConvTranspose(inner_channels // 4, inner_channels // 4, 2, 2), + BatchNorm2d(inner_channels // 4), nn.ReLU(inplace=True), + DwPwConvTranspose(inner_channels // 4, 1, 2, 2), nn.Sigmoid()) + + self.adaptive = adaptive + if adaptive: + self.thresh = self._init_thresh( + inner_channels, serial=serial, smooth=smooth, bias=bias) + + # weight initialization + for m in self.modules(): + if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d): + nn.init.kaiming_normal_(m.weight) + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1.) + m.bias.data.fill_(1e-4) + + def _init_thresh(self, + inner_channels, + serial=False, + smooth=False, + bias=False): + in_channels = inner_channels + if serial: + in_channels += 1 + self.thresh = nn.Sequential( + nn.Conv2d( + inner_channels, + inner_channels // 4, + self.dw_kernel_size, + padding=self.dw_padding, + bias=bias), BatchNorm2d(inner_channels // 4), + nn.ReLU(inplace=True), + self._init_upsample( + inner_channels // 4, + inner_channels // 4, + smooth=smooth, + bias=bias), BatchNorm2d(inner_channels // 4), + nn.ReLU(inplace=True), + self._init_upsample( + inner_channels // 4, 1, smooth=smooth, bias=bias), + nn.Sigmoid()) + + return self.thresh + + def _init_upsample(self, + in_channels, + out_channels, + smooth=False, + bias=False): + if smooth: + inter_out_channels = out_channels + if out_channels == 1: + inter_out_channels = in_channels + module_list = [ + nn.Upsample(scale_factor=2, mode='nearest'), + nn.Conv2d(in_channels, inter_out_channels, 3, 1, 1, bias=bias) + ] + if out_channels == 1: + module_list.append( + nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=1, + bias=True)) + + return nn.Sequential(module_list) + else: + return nn.ConvTranspose2d(in_channels, out_channels, 2, 2) + + def forward(self, features, gt=None, masks=None, training=False): + c2, c3, c4, c5 = features + p5 = self.up5(self.in5(c5)) + p4 = self.up4(self.in4(c4)) + p3 = self.up3(self.in3(c3)) + p2 = self.in2(c2) + + fuse = p5 + p4 + p3 + p2 + + # this is the pred module, not binarization module; + # We do not correct the name due to the trained model. + binary = self.binarize(fuse) + if self.training: + result = OrderedDict(binary=binary) + else: + return binary + if self.adaptive and self.training: + if self.serial: + fuse = torch.cat( + (fuse, nn.functional.interpolate(binary, fuse.shape[2:])), + 1) + thresh = self.thresh(fuse) + thresh_binary = self.step_function(binary, thresh) + result.update(thresh=thresh, thresh_binary=thresh_binary) + return result + + def step_function(self, x, y): + return torch.reciprocal(1 + torch.exp(-self.k * (x - y))) + + class SegDetector(nn.Module): def __init__(self, @@ -471,6 +690,28 @@ class VLPTModel(nn.Module): return self.decoder(self.backbone(x)) +class DBNasModel(nn.Module): + + def __init__(self, *args, **kwargs): + """ + DB-NAS model + """ + super(DBNasModel, self).__init__() + self.backbone = CompactDetBackbone( + width_stages=[32, 64, 96, 128], input_channel=32, **kwargs) + self.decoder = LightSegDetector( + in_channels=[32, 64, 96, 128], + adaptive=True, + k=50, + inner_channels=64, + dw_kernel_size=5, + dw_padding=2, + **kwargs) + + def forward(self, x): + return self.decoder(self.backbone(x)) + + class DBModel(nn.Module): def __init__(self, *args, **kwargs): diff --git a/modelscope/models/cv/ocr_detection/modules/layers.py b/modelscope/models/cv/ocr_detection/modules/layers.py new file mode 100644 index 00000000..515c0bc6 --- /dev/null +++ b/modelscope/models/cv/ocr_detection/modules/layers.py @@ -0,0 +1,906 @@ +from collections import OrderedDict + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def get_same_padding(kernel_size): + if isinstance(kernel_size, tuple): + assert len(kernel_size) == 2, 'invalid kernel size: %s' % kernel_size + p1 = get_same_padding(kernel_size[0]) + p2 = get_same_padding(kernel_size[1]) + return p1, p2 + assert isinstance(kernel_size, + int), 'kernel size should be either `int` or `tuple`' + assert kernel_size % 2 > 0, 'kernel size should be odd number' + return kernel_size // 2 + + +def count_conv_flop(layer, x): + out_h = int(x.size(2) / layer.stride[0]) + out_w = int(x.size(3) / layer.stride[1]) + delta_ops = layer.in_channels * layer.out_channels * layer.kernel_size[ + 0] * layer.kernel_size[1] * out_h * out_w / layer.groups + return delta_ops + + +def set_layer_from_config(layer_config): + if layer_config is None: + return None + name2layer = { + ZeroLayer.__name__: ZeroLayer, + MBInvertedConvLayer.__name__: MBInvertedConvLayer, + IdentityLayer.__name__: IdentityLayer, + } + layer_name = layer_config.pop('name') + layer = name2layer[layer_name] + return layer.build_from_config(layer_config) + + +class MobileInvertedResidualBlock(nn.Module): + + def __init__(self, mobile_inverted_conv, shortcut): + super(MobileInvertedResidualBlock, self).__init__() + self.mobile_inverted_conv = mobile_inverted_conv + self.shortcut = shortcut + + def forward(self, x): + if self.mobile_inverted_conv.is_zero_layer(): + res = x + elif self.shortcut is None or self.shortcut.is_zero_layer(): + res = self.mobile_inverted_conv(x) + else: + conv_x = self.mobile_inverted_conv(x) + skip_x = self.shortcut(x) + res = skip_x + conv_x + return res + + @property + def module_str(self): + return '(%s, %s)' % (self.mobile_inverted_conv.module_str, + self.shortcut.module_str + if self.shortcut is not None else None) + + @property + def config(self): + return { + 'name': + MobileInvertedResidualBlock.__name__, + 'mobile_inverted_conv': + self.mobile_inverted_conv.config, + 'shortcut': + self.shortcut.config if self.shortcut is not None else None, + } + + @staticmethod + def build_from_config(config): + mobile_inverted_conv = set_layer_from_config( + config['mobile_inverted_conv']) + shortcut = set_layer_from_config(config['shortcut']) + return MobileInvertedResidualBlock(mobile_inverted_conv, shortcut) + + def get_flops(self, x): + flops1, _ = self.mobile_inverted_conv.get_flops(x) + if self.shortcut: + flops2, _ = self.shortcut.get_flops(x) + else: + flops2 = 0 + return flops1 + flops2, self.forward(x) + + +class MBInvertedConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + stride=(1, 1), + expand_ratio=6, + mid_channels=None): + super(MBInvertedConvLayer, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.stride = stride + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + + feature_dim = round( + self.in_channels + * self.expand_ratio) if mid_channels is None else mid_channels + if self.expand_ratio == 1: + self.inverted_bottleneck = None + else: + self.inverted_bottleneck = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + self.in_channels, feature_dim, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + pad = get_same_padding(self.kernel_size) + self.depth_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + feature_dim, + feature_dim, + kernel_size, + stride, + pad, + groups=feature_dim, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + def forward(self, x): + if self.inverted_bottleneck: + x = self.inverted_bottleneck(x) + x = self.depth_conv(x) + x = self.point_conv(x) + return x + + @property + def module_str(self): + return '%dx%d_MBConv%d' % (self.kernel_size, self.kernel_size, + self.expand_ratio) + + @property + def config(self): + return { + 'name': MBInvertedConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'kernel_size': self.kernel_size, + 'stride': self.stride, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return MBInvertedConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + if self.inverted_bottleneck: + total_flops += count_conv_flop(self.inverted_bottleneck.conv, x) + x = self.inverted_bottleneck(x) + total_flops += count_conv_flop(self.depth_conv.conv, x) + x = self.depth_conv(x) + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x + + +class IdentityLayer(nn.Module): + + def __init__(self): + super(IdentityLayer, self).__init__() + + def forward(self, x): + return x + + @property + def module_str(self): + return 'Identity' + + @property + def config(self): + return { + 'name': IdentityLayer.__name__, + } + + @staticmethod + def build_from_config(config): + return IdentityLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + return 0, self.forward(x) + + +class ZeroLayer(nn.Module): + + def __init__(self, stride): + super(ZeroLayer, self).__init__() + self.stride = stride + + def forward(self, x): + n, c, h, w = x.shape + h //= self.stride[0] + w //= self.stride[1] + device = x.device + padding = torch.zeros(n, c, h, w, device=device, requires_grad=False) + return padding + + @property + def module_str(self): + return 'Zero' + + @property + def config(self): + return {'name': ZeroLayer.__name__, 'stride': self.stride} + + @staticmethod + def build_from_config(config): + return ZeroLayer(**config) + + @staticmethod + def is_zero_layer(): + return True + + def get_flops(self, x): + return 0, self.forward(x) + + +def split_layer(total_channels, num_groups): + split = [ + int(np.ceil(total_channels / num_groups)) for _ in range(num_groups) + ] + split[num_groups - 1] += total_channels - sum(split) + return split + + +class MBInvertedMixConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + mix_conv_size=[1, 3, 5], + stride=(1, 1), + expand_ratio=6, + mid_channels=None): + super(MBInvertedMixConvLayer, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.mix_conv_size = mix_conv_size + self.stride = stride + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + + feature_dim = round( + self.in_channels + * self.expand_ratio) if mid_channels is None else mid_channels + self.inverted_bottleneck = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(self.in_channels, feature_dim, 1, 1, 0, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + + self.mix_conv_size = mix_conv_size + self.n_chunks = len(mix_conv_size) + self.split_in_channels = split_layer(feature_dim, self.n_chunks) + + self.mix_conv = nn.ModuleList() + for idx in range(self.n_chunks): + kernel_size = self.mix_conv_size[idx] + pad = get_same_padding(kernel_size) + split_in_channels_ = self.split_in_channels[idx] + self.mix_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + split_in_channels_, + split_in_channels_, + kernel_size, + stride, + pad, + groups=split_in_channels_, + bias=False)), + ('bn', nn.BatchNorm2d(split_in_channels_)), + ('act', nn.PReLU()), + ]))) + + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + def forward(self, x): + x = self.inverted_bottleneck(x) + split = torch.split(x, self.split_in_channels, dim=1) + x = torch.cat([layer(s) for layer, s in zip(self.mix_conv, split)], + dim=1) + + x = self.point_conv(x) + return x + + @property + def module_str(self): + return '%s_MixConv%d' % (str(self.mix_conv_size), self.expand_ratio) + + @property + def config(self): + return { + 'name': MBInvertedMixConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'mix_conv_size': self.mix_conv_size, + 'stride': self.stride, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return MBInvertedMixConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + total_flops += count_conv_flop(self.inverted_bottleneck.conv, x) + x = self.inverted_bottleneck(x) + split = torch.split(x, self.split_in_channels, dim=1) + out = [] + for layer, s in zip(self.mix_conv, split): + out.append(layer(s)) + total_flops += count_conv_flop(layer.conv, s) + x = torch.cat(out, dim=1) + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x + + +class LinearMixConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + mix_conv_size=[1, 3, 5], + stride=(1, 1), + mid_channels=None): + super(LinearMixConvLayer, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.mix_conv_size = mix_conv_size + self.stride = stride + self.mid_channels = mid_channels + + self.mix_conv_size = mix_conv_size + self.n_chunks = len(mix_conv_size) + self.mix_conv = nn.ModuleList() + + for idx in range(self.n_chunks): + kernel_size = self.mix_conv_size[idx] + pad = get_same_padding(kernel_size) + self.mix_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + in_channels, + in_channels, + kernel_size, + stride, + pad, + groups=in_channels, + bias=False)), + ('bn', nn.BatchNorm2d(in_channels)), + ('act', nn.ReLU6(inplace=True)), + ]))) + + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + in_channels * self.n_chunks, + out_channels, + 1, + 1, + 0, + bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + def forward(self, x): + x = torch.cat([layer(x) for layer in self.mix_conv], dim=1) + + x = self.point_conv(x) + return x + + @property + def module_str(self): + return '%s_LinearMixConv' % (str(self.mix_conv_size)) + + @property + def config(self): + return { + 'name': LinearMixConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'mix_conv_size': self.mix_conv_size, + 'stride': self.stride, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return LinearMixConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + out = [] + for layer in self.mix_conv: + out.append(layer(x)) + total_flops += count_conv_flop(layer.conv, x) + x = torch.cat(out, dim=1) + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x + + +class SELayer(nn.Module): + ''' + ''' + + def __init__(self, input_channels: int, squeeze_factor: int = 4): + super(SELayer, self).__init__() + self.input_channels = input_channels + self.squeeze_factor = squeeze_factor + self.squeeze_channels = input_channels // squeeze_factor + self.fc1 = nn.Conv2d(self.input_channels, self.squeeze_channels, 1) + self.relu = nn.ReLU(inplace=True) + self.fc2 = nn.Conv2d(self.squeeze_channels, self.input_channels, 1) + + def _scale(self, input): + scale = F.adaptive_avg_pool2d(input, 1) + scale = self.fc1(scale) + scale = self.relu(scale) + scale = self.fc2(scale) + return torch.sigmoid(scale) + + def forward(self, input): + scale = self._scale(input) + return scale * input + + @property + def module_str(self): + return 'SE_%d' % (self.squeeze_factor) + + @property + def config(self): + return { + 'name': SELayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'squeeze_factor': self.squeeze_factor, + } + + @staticmethod + def build_from_config(config): + return SELayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + ''' + count se flops, only compute the fc layers' calculation + ''' + total_flops = 0 + total_flops += self.input_channels * self.squeeze_channels * 2 + b, c, h, w = x.shape + total_flops += c * h * w + return total_flops, x + + +class MHSA(nn.Module): + + def __init__(self, n_dims, width=14, height=14, heads=4): + super(MHSA, self).__init__() + self.heads = heads + + self.query = nn.Conv2d(n_dims, n_dims, kernel_size=1) + self.key = nn.Conv2d(n_dims, n_dims, kernel_size=1) + self.value = nn.Conv2d(n_dims, n_dims, kernel_size=1) + + self.width = width + self.height = height + + self.rel_h = nn.Parameter( + torch.randn([1, heads, n_dims // heads, 1, height]), + requires_grad=True) + self.rel_w = nn.Parameter( + torch.randn([1, heads, n_dims // heads, width, 1]), + requires_grad=True) + + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x): + + n_batch, C, width, height = x.size() + q = self.query(x).view(n_batch, self.heads, C // self.heads, -1) + k = self.key(x).view(n_batch, self.heads, C // self.heads, -1) + v = self.value(x).view(n_batch, self.heads, C // self.heads, -1) + + content_content = torch.matmul(q.permute(0, 1, 3, 2), k) + + content_position = (self.rel_h + self.rel_w).view( + 1, self.heads, C // self.heads, -1).permute(0, 1, 3, 2) + content_position = torch.matmul(content_position, q) + energy = content_content + content_position + attention = self.softmax(energy) + + out = torch.matmul(v, attention.permute(0, 1, 3, 2)) + out = out.view(n_batch, C, width, height) + + return out + + def get_flops(self, x): + ''' + count se flops, only compute the fc layers' calculation + ''' + n_batch, C, width, height = x.size() + + total_flops = 0 + total_flops += count_conv_flop(self.query, x) * 3 + + # content_content + total_flops += (width * height) * C * (width * height) + + # content_position + total_flops += (width * height) * C + total_flops += (width * height) * C * (width * height) + + # attention + total_flops += (width * height) * C * (width * height) + + return total_flops, x + + +class MBInvertedMHSALayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + expand_ratio=6, + width=1, + height=175, + mid_channels=None): + super(MBInvertedMHSALayer, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + + feature_dim = round( + self.in_channels + * self.expand_ratio) if mid_channels is None else mid_channels + if self.expand_ratio == 1: + self.inverted_bottleneck = None + else: + self.inverted_bottleneck = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + self.in_channels, feature_dim, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + # ('act', nn.PReLU()), + ('act', nn.ReLU6(inplace=True)), + ])) + self.mhsa = MHSA(feature_dim, width, height) + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + def forward(self, x): + if self.inverted_bottleneck: + x = self.inverted_bottleneck(x) + x = self.mhsa(x) + x = self.point_conv(x) + return x + + @property + def module_str(self): + return 'MSHA%d' % (self.expand_ratio) + + @property + def config(self): + return { + 'name': MBInvertedMHSALayer.__name__, + 'in_channels': self.in_channels, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return MBInvertedMHSALayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + if self.inverted_bottleneck: + total_flops += count_conv_flop(self.inverted_bottleneck.conv, x) + x = self.inverted_bottleneck(x) + total_flops += self.mhsa.get_flops(x)[0] + x = self.mhsa(x) + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x + + +class MBInvertedRepConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + rep_conv_size=[1, 3, 5], + stride=(1, 1), + expand_ratio=6, + mid_channels=None): + super(MBInvertedRepConvLayer, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.rep_conv_size = rep_conv_size + self.stride = stride + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + + feature_dim = round( + self.in_channels + * self.expand_ratio) if mid_channels is None else mid_channels + + self.inverted_bottleneck = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(self.in_channels, feature_dim, 1, 1, 0, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + + self.rep_conv_size = rep_conv_size + self.n_chunks = len(rep_conv_size) + + self.rep_conv = nn.ModuleList() + for idx in range(self.n_chunks): + kernel_size = self.rep_conv_size[idx] + pad = get_same_padding(kernel_size) + self.rep_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + feature_dim, + feature_dim, + kernel_size, + stride, + pad, + groups=feature_dim, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ]))) + + self.rep_conv_deploy = None + self.act = nn.PReLU() + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + self.deploy = False + + def forward(self, x): + x = self.inverted_bottleneck(x) + if not self.deploy: + out = [] + for layer in self.rep_conv: + out.append(layer(x)) + x = out[0] + for out_ in out[1:]: + x += out_ + else: + x = self.rep_conv_deploy(x) + x = self.act(x) + x = self.point_conv(x) + return x + + @property + def module_str(self): + return '%s_RepConv%d' % (str(self.rep_conv_size), self.expand_ratio) + + @property + def config(self): + return { + 'name': MBInvertedMixConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'rep_conv_size': self.rep_conv_size, + 'stride': self.stride, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return MBInvertedMixConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def switch_to_deploy(self): + self.deploy = True + feature_dim = self.rep_conv[0].conv.in_channels + stride = self.rep_conv[0].conv.stride + kernel_size = max(self.rep_conv_size) + pad = get_same_padding(kernel_size) + self.rep_conv_deploy = nn.Conv2d( + feature_dim, + feature_dim, + kernel_size, + stride, + pad, + groups=feature_dim, + bias=True) + + kernel, bias = self.get_equivalent_kernel_bias() + + self.rep_conv_deploy.weight.data = kernel + self.rep_conv_deploy.bias.data = bias + for para in self.parameters(): + para.detach_() + self.__delattr__('rep_conv') + + def get_equivalent_kernel_bias(self): + max_kernel_size = max(self.rep_conv_size) + + if max_kernel_size == 5: + if 1 in self.rep_conv_size: + kernel1x1, bias1x1 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(1)]) + else: + kernel1x1 = None + bias1x1 = 0 + + if 3 in self.rep_conv_size: + kernel3x3, bias3x3 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(3)]) + else: + kernel3x3 = None + bias3x3 = 0 + + if 5 in self.rep_conv_size: + kernel5x5, bias5x5 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(5)]) + + else: + kernel5x5 = 0 + bias5x5 = 0 + + return kernel5x5 + self._pad_1x1_to_5x5_tensor( + kernel1x1) + self._pad_3x3_to_5x5_tensor( + kernel3x3), bias5x5 + bias3x3 + bias1x1 + else: + if 1 in self.rep_conv_size: + kernel1x1, bias1x1 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(1)]) + else: + kernel1x1 = None + bias1x1 = 0 + + if 3 in self.rep_conv_size: + kernel3x3, bias3x3 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(3)]) + else: + kernel3x3 = None + bias3x3 = 0 + + return kernel3x3 + self._pad_1x1_to_3x3_tensor( + kernel1x1), bias3x3 + bias1x1 + + def _pad_1x1_to_3x3_tensor(self, kernel1x1): + if kernel1x1 is None: + return 0 + else: + return torch.nn.functional.pad(kernel1x1, [1, 1, 1, 1]) + + def _pad_1x1_to_5x5_tensor(self, kernel1x1): + if kernel1x1 is None: + return 0 + else: + return torch.nn.functional.pad(kernel1x1, [2, 2, 2, 2]) + + def _pad_3x3_to_5x5_tensor(self, kernel3x3): + if kernel3x3 is None: + return 0 + else: + return torch.nn.functional.pad(kernel3x3, [1, 1, 1, 1]) + + def _fuse_bn_tensor(self, branch): + if branch is None: + return 0, 0 + if isinstance(branch, nn.Sequential): + kernel = branch.conv.weight + running_mean = branch.bn.running_mean + running_var = branch.bn.running_var + gamma = branch.bn.weight + beta = branch.bn.bias + eps = branch.bn.eps + else: + assert isinstance(branch, nn.BatchNorm2d) + if not hasattr(self, 'id_tensor'): + input_dim = self.in_channels // self.groups + kernel_value = np.zeros((self.in_channels, input_dim, 3, 3), + dtype=np.float32) + for i in range(self.in_channels): + kernel_value[i, i % input_dim, 1, 1] = 1 + self.id_tensor = torch.from_numpy(kernel_value).to( + branch.weight.device) + kernel = self.id_tensor + running_mean = branch.running_mean + running_var = branch.running_var + gamma = branch.weight + beta = branch.bias + eps = branch.eps + std = (running_var + eps).sqrt() + t = (gamma / std).reshape(-1, 1, 1, 1) + + return kernel * t, beta - running_mean * gamma / std + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + total_flops += count_conv_flop(self.inverted_bottleneck.conv, x) + x = self.inverted_bottleneck(x) + + total_flops += count_conv_flop(self.rep_conv[-1].conv, x) + out = [] + for layer in self.rep_conv: + out.append(layer(x)) + x = out[0] + for out_ in out[1:]: + x += out_ + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x diff --git a/modelscope/models/cv/ocr_detection/modules/mix_ops.py b/modelscope/models/cv/ocr_detection/modules/mix_ops.py new file mode 100644 index 00000000..5c6dd4a5 --- /dev/null +++ b/modelscope/models/cv/ocr_detection/modules/mix_ops.py @@ -0,0 +1,689 @@ +# ------------------------------------------------------------------------------ +# Part of implementation is adopted from ProxylessNAS, +# made publicly available under the Apache License 2.0 at https://github.com/mit-han-lab/proxylessnas. +# ------------------------------------------------------------------------------ + +import math + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.parameter import Parameter + +from .layers import (IdentityLayer, LinearMixConvLayer, MBInvertedConvLayer, + MBInvertedMHSALayer, MBInvertedMixConvLayer, + MBInvertedRepConvLayer, SELayer, ZeroLayer) + + +def detach_variable(inputs): + if isinstance(inputs, tuple): + return tuple([detach_variable(x) for x in inputs]) + else: + x = inputs.detach() + x.requires_grad = inputs.requires_grad + return x + + +def delta_ij(i, j): + if i == j: + return 1 + else: + return 0 + + +def conv_func_by_name(name): + name2ops = { + 'Identity': + lambda in_C, out_C, S: IdentityLayer(in_C, out_C, ops_order=ops_order), + 'Zero': + lambda in_C, out_C, S: ZeroLayer(stride=S), + } + name2ops.update({ + '3x3_MBConv1': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 1), + '3x3_MBConv2': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 2), + '3x3_MBConv3': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 3), + '3x3_MBConv4': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 4), + '3x3_MBConv5': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 5), + '3x3_MBConv6': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 6), + ####################################################################################### + '5x5_MBConv1': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 1), + '5x5_MBConv2': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 2), + '5x5_MBConv3': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 3), + '5x5_MBConv4': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 4), + '5x5_MBConv5': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 5), + '5x5_MBConv6': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 6), + ####################################################################################### + '7x7_MBConv1': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 1), + '7x7_MBConv2': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 2), + '7x7_MBConv3': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 3), + '7x7_MBConv4': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 4), + '7x7_MBConv5': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 5), + '7x7_MBConv6': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 6), + ####################################################################################### + '13_MixConv1': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 1 + ), + '13_MixConv2': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 2 + ), + '13_MixConv3': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 3 + ), + '13_MixConv4': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 4 + ), + '13_MixConv5': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 5 + ), + '13_MixConv6': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 6 + ), + ####################################################################################### + '35_MixConv1': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 1 + ), + '35_MixConv2': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 2 + ), + '35_MixConv3': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 3 + ), + '35_MixConv4': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 4 + ), + '35_MixConv5': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 5 + ), + '35_MixConv6': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 6 + ), + ####################################################################################### + '135_MixConv1': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 1), + '135_MixConv2': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 2), + '135_MixConv3': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 3), + '135_MixConv4': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 4), + '135_MixConv5': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 5), + '135_MixConv6': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 6), + ####################################################################################### + '13_LinMixConv': + lambda in_C, out_C, S: LinearMixConvLayer(in_C, out_C, [1, 3], S), + '35_LinMixConv': + lambda in_C, out_C, S: LinearMixConvLayer(in_C, out_C, [3, 5], S), + '135_LinMixConv': + lambda in_C, out_C, S: LinearMixConvLayer(in_C, out_C, [1, 3, 5], S), + ####################################################################################### + 'SE_2': + lambda in_C, out_C, S: SELayer(in_C, 2), + 'SE_4': + lambda in_C, out_C, S: SELayer(in_C, 4), + 'SE_8': + lambda in_C, out_C, S: SELayer(in_C, 8), + ####################################################################################### + 'MHSA1': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 1, width, height), + 'MHSA2': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 2, width, height), + 'MHSA3': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 3, width, height), + 'MHSA4': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 4, width, height), + 'MHSA5': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 5, width, height), + 'MHSA6': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 6, width, height), + ####################################################################################### + '13_RepConv1': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 1 + ), + '13_RepConv2': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 2 + ), + '13_RepConv3': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 3 + ), + '13_RepConv4': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 4 + ), + '13_RepConv5': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 5 + ), + '13_RepConv6': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 6 + ), + ####################################################################################### + '35_RepConv1': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 1 + ), + '35_RepConv2': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 2 + ), + '35_RepConv3': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 3 + ), + '35_RepConv4': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 4 + ), + '35_RepConv5': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 5 + ), + '35_RepConv6': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 6 + ), + ####################################################################################### + '135_RepConv1': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 1), + '135_RepConv2': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 2), + '135_RepConv3': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 3), + '135_RepConv4': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 4), + '135_RepConv5': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 5), + '135_RepConv6': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 6), + }) + return name2ops[name] + + +def build_candidate_ops(candidate_ops, + in_channels, + out_channels, + stride, + ops_order, + spatio_size=None): + if candidate_ops is None: + raise ValueError('please specify a candidate set') + + name2ops = { + 'Identity': + lambda in_C, out_C, S: IdentityLayer(in_C, out_C, ops_order=ops_order), + 'Zero': + lambda in_C, out_C, S: ZeroLayer(stride=S), + } + # add MBConv layers + name2ops.update({ + '3x3_MBConv1': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 1), + '3x3_MBConv2': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 2), + '3x3_MBConv3': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 3), + '3x3_MBConv4': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 4), + '3x3_MBConv5': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 5), + '3x3_MBConv6': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 6), + ####################################################################################### + '5x5_MBConv1': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 1), + '5x5_MBConv2': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 2), + '5x5_MBConv3': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 3), + '5x5_MBConv4': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 4), + '5x5_MBConv5': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 5), + '5x5_MBConv6': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 6), + ####################################################################################### + '7x7_MBConv1': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 1), + '7x7_MBConv2': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 2), + '7x7_MBConv3': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 3), + '7x7_MBConv4': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 4), + '7x7_MBConv5': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 5), + '7x7_MBConv6': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 6), + ####################################################################################### + '13_MixConv1': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 1 + ), + '13_MixConv2': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 2 + ), + '13_MixConv3': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 3 + ), + '13_MixConv4': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 4 + ), + '13_MixConv5': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 5 + ), + '13_MixConv6': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 6 + ), + ####################################################################################### + '35_MixConv1': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 1 + ), + '35_MixConv2': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 2 + ), + '35_MixConv3': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 3 + ), + '35_MixConv4': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 4 + ), + '35_MixConv5': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 5 + ), + '35_MixConv6': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 6 + ), + ####################################################################################### + '135_MixConv1': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 1), + '135_MixConv2': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 2), + '135_MixConv3': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 3), + '135_MixConv4': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 4), + '135_MixConv5': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 5), + '135_MixConv6': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 6), + ####################################################################################### + '13_LinMixConv': + lambda in_C, out_C, S: LinearMixConvLayer(in_C, out_C, [1, 3], S), + '35_LinMixConv': + lambda in_C, out_C, S: LinearMixConvLayer(in_C, out_C, [3, 5], S), + '135_LinMixConv': + lambda in_C, out_C, S: LinearMixConvLayer(in_C, out_C, [1, 3, 5], S), + ####################################################################################### + 'SE_2': + lambda in_C, out_C, S: SELayer(in_C, 2), + 'SE_4': + lambda in_C, out_C, S: SELayer(in_C, 4), + 'SE_8': + lambda in_C, out_C, S: SELayer(in_C, 8), + ####################################################################################### + 'MHSA1': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 1, width, height), + 'MHSA2': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 2, width, height), + 'MHSA3': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 3, width, height), + 'MHSA4': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 4, width, height), + 'MHSA5': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 5, width, height), + 'MHSA6': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 6, width, height), + ####################################################################################### + '13_RepConv1': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 1 + ), + '13_RepConv2': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 2 + ), + '13_RepConv3': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 3 + ), + '13_RepConv4': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 4 + ), + '13_RepConv5': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 5 + ), + '13_RepConv6': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 6 + ), + ####################################################################################### + '35_RepConv1': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 1 + ), + '35_RepConv2': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 2 + ), + '35_RepConv3': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 3 + ), + '35_RepConv4': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 4 + ), + '35_RepConv5': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 5 + ), + '35_RepConv6': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 6 + ), + ####################################################################################### + '135_RepConv1': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 1), + '135_RepConv2': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 2), + '135_RepConv3': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 3), + '135_RepConv4': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 4), + '135_RepConv5': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 5), + '135_RepConv6': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 6), + }) + + out = [] + for name in candidate_ops: + + if 'MHSA' in name: + out.append(name2ops[name](in_channels, out_channels, + spatio_size[0], spatio_size[1])) + else: + out.append(name2ops[name](in_channels, out_channels, stride)) + return out + + +class MixedEdge(nn.Module): + MODE = None # full, two, None, full_v2 + + def __init__(self, candidate_ops): + super(MixedEdge, self).__init__() + + self.candidate_ops = nn.ModuleList(candidate_ops) + self.AP_path_alpha = Parameter(torch.Tensor( + self.n_choices)) # architecture parameters + self.AP_path_wb = Parameter(torch.Tensor( + self.n_choices)) # binary gates + + self.active_index = [0] + self.inactive_index = None + + self.log_prob = None + self.current_prob_over_ops = None + + @property + def n_choices(self): + return len(self.candidate_ops) + + @property + def probs_over_ops(self): + probs = F.softmax(self.AP_path_alpha, dim=0) # softmax to probability + return probs + + @property + def chosen_index(self): + probs = self.probs_over_ops.data.cpu().numpy() + index = int(np.argmax(probs)) + return index, probs[index] + + @property + def chosen_op(self): + index, _ = self.chosen_index + return self.candidate_ops[index] + + @property + def random_op(self): + index = np.random.choice([_i for _i in range(self.n_choices)], 1)[0] + return self.candidate_ops[index] + + def entropy(self, eps=1e-8): + probs = self.probs_over_ops + log_probs = torch.log(probs + eps) + entropy = -torch.sum(torch.mul(probs, log_probs)) + return entropy + + def is_zero_layer(self): + return self.active_op.is_zero_layer() + + @property + def active_op(self): + """ assume only one path is active """ + return self.candidate_ops[self.active_index[0]] + + def set_chosen_op_active(self): + chosen_idx, _ = self.chosen_index + self.active_index = [chosen_idx] + self.inactive_index = [_i for _i in range(0, chosen_idx)] + \ + [_i for _i in range(chosen_idx + 1, self.n_choices)] + + def forward(self, x): + if MixedEdge.MODE == 'full' or MixedEdge.MODE == 'two': + output = 0 + for _i in self.active_index: + oi = self.candidate_ops[_i](x) + output = output + self.AP_path_wb[_i] * oi + for _i in self.inactive_index: + oi = self.candidate_ops[_i](x) + output = output + self.AP_path_wb[_i] * oi.detach() + elif MixedEdge.MODE == 'full_v2': + + def run_function(candidate_ops, active_id): + + def forward(_x): + return candidate_ops[active_id](_x) + + return forward + + def backward_function(candidate_ops, active_id, binary_gates): + + def backward(_x, _output, grad_output): + binary_grads = torch.zeros_like(binary_gates.data) + with torch.no_grad(): + for k in range(len(candidate_ops)): + if k != active_id: + out_k = candidate_ops[k](_x.data) + else: + out_k = _output.data + grad_k = torch.sum(out_k * grad_output) + binary_grads[k] = grad_k + return binary_grads + + return backward + + output = ArchGradientFunction.apply( + x, self.AP_path_wb, + run_function(self.candidate_ops, self.active_index[0]), + backward_function(self.candidate_ops, self.active_index[0], + self.AP_path_wb)) + else: + output = self.active_op(x) + return output + + @property + def module_str(self): + chosen_index, probs = self.chosen_index + return 'Mix(%s, %.3f)' % (self.candidate_ops[chosen_index].module_str, + probs) + + @property + def config(self): + raise ValueError('not needed') + + @staticmethod + def build_from_config(config): + raise ValueError('not needed') + + def get_flops(self, x): + """ Only active paths taken into consideration when calculating FLOPs """ + flops = 0 + for i in self.active_index: + delta_flop, _ = self.candidate_ops[i].get_flops(x) + flops += delta_flop + return flops, self.forward(x) + + def binarize(self): + """ prepare: active_index, inactive_index, AP_path_wb, log_prob (optional), current_prob_over_ops (optional) """ + self.log_prob = None + # reset binary gates + self.AP_path_wb.data.zero_() + # binarize according to probs + probs = self.probs_over_ops + if MixedEdge.MODE == 'two': + # sample two ops according to `probs` + sample_op = torch.multinomial(probs.data, 2, replacement=False) + probs_slice = F.softmax( + torch.stack([self.AP_path_alpha[idx] for idx in sample_op]), + dim=0) + self.current_prob_over_ops = torch.zeros_like(probs) + for i, idx in enumerate(sample_op): + self.current_prob_over_ops[idx] = probs_slice[i] + # chose one to be active and the other to be inactive according to probs_slice + c = torch.multinomial(probs_slice.data, 1)[0] # 0 or 1 + active_op = sample_op[c].item() + inactive_op = sample_op[1 - c].item() + self.active_index = [active_op] + self.inactive_index = [inactive_op] + # set binary gate + self.AP_path_wb.data[active_op] = 1.0 + else: + sample = torch.multinomial(probs.data, 1)[0].item() + self.active_index = [sample] + self.inactive_index = [_i for _i in range(0, sample)] + \ + [_i for _i in range(sample + 1, self.n_choices)] + self.log_prob = torch.log(probs[sample]) + self.current_prob_over_ops = probs + # set binary gate + self.AP_path_wb.data[sample] = 1.0 + # avoid over-regularization + for _i in range(self.n_choices): + for name, param in self.candidate_ops[_i].named_parameters(): + param.grad = None + + def set_arch_param_grad(self): + binary_grads = self.AP_path_wb.grad.data + if self.active_op.is_zero_layer(): + self.AP_path_alpha.grad = None + return + if self.AP_path_alpha.grad is None: + self.AP_path_alpha.grad = torch.zeros_like(self.AP_path_alpha.data) + if MixedEdge.MODE == 'two': + involved_idx = self.active_index + self.inactive_index + probs_slice = F.softmax( + torch.stack([self.AP_path_alpha[idx] for idx in involved_idx]), + dim=0).data + for i in range(2): + for j in range(2): + origin_i = involved_idx[i] + origin_j = involved_idx[j] + self.AP_path_alpha.grad.data[origin_i] += \ + binary_grads[origin_j] * probs_slice[j] * (delta_ij(i, j) - probs_slice[i]) + for _i, idx in enumerate(self.active_index): + self.active_index[_i] = (idx, + self.AP_path_alpha.data[idx].item()) + for _i, idx in enumerate(self.inactive_index): + self.inactive_index[_i] = (idx, + self.AP_path_alpha.data[idx].item()) + else: + probs = self.probs_over_ops.data + for i in range(self.n_choices): + for j in range(self.n_choices): + self.AP_path_alpha.grad.data[ + i] += binary_grads[j] * probs[j] * ( + delta_ij(i, j) - probs[i]) + return + + def rescale_updated_arch_param(self): + if not isinstance(self.active_index[0], tuple): + assert self.active_op.is_zero_layer() + return + involved_idx = [ + idx for idx, _ in (self.active_index + self.inactive_index) + ] + old_alphas = [ + alpha for _, alpha in (self.active_index + self.inactive_index) + ] + new_alphas = [self.AP_path_alpha.data[idx] for idx in involved_idx] + + offset = math.log( + sum([math.exp(alpha) for alpha in new_alphas]) + / sum([math.exp(alpha) for alpha in old_alphas])) + + for idx in involved_idx: + self.AP_path_alpha.data[idx] -= offset + + +class ArchGradientFunction(torch.autograd.Function): + + @staticmethod + def forward(ctx, x, binary_gates, run_func, backward_func): + ctx.run_func = run_func + ctx.backward_func = backward_func + + detached_x = detach_variable(x) + with torch.enable_grad(): + output = run_func(detached_x) + ctx.save_for_backward(detached_x, output) + return output.data + + @staticmethod + def backward(ctx, grad_output): + detached_x, output = ctx.saved_tensors + + grad_x = torch.autograd.grad( + output, detached_x, grad_output, only_inputs=True) + # compute gradients w.r.t. binary_gates + binary_grads = ctx.backward_func(detached_x.data, output.data, + grad_output.data) + + return grad_x[0], binary_grads, None, None diff --git a/modelscope/models/cv/ocr_detection/modules/proxyless.py b/modelscope/models/cv/ocr_detection/modules/proxyless.py new file mode 100644 index 00000000..68458bc3 --- /dev/null +++ b/modelscope/models/cv/ocr_detection/modules/proxyless.py @@ -0,0 +1,178 @@ +import re +import sys + +import numpy as np +import torch +import torch.nn as nn + +from .layers import (IdentityLayer, MBInvertedConvLayer, + MobileInvertedResidualBlock, ZeroLayer) +from .mix_ops import MixedEdge, build_candidate_ops, conv_func_by_name + + +class NasRecBackbone(nn.Module): + + def __init__(self, first_conv, blocks): + super(NasRecBackbone, self).__init__() + self.first_conv = first_conv + self.blocks = nn.ModuleList(blocks) + self.output_idx = [5, 11, 17, 23] + + def forward(self, x): + x = self.first_conv(x) + idx = 0 + out = [] + for block in self.blocks: + x = block(x) + if (idx + 1) % (int(len(self.blocks) / 4)) == 0: + out.append(x) + idx += 1 + return out[0], out[1], out[2], out[3] + + def get_bn_param(self): + for m in self.modules(): + if isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.BatchNorm1d): + return { + 'momentum': m.momentum, + 'eps': m.eps, + } + return None + + @property + def config(self): + return { + 'name': NasRecBackbone.__name__, + 'bn': self.get_bn_param(), + 'first_conv': 'conv_in3_out32_k3_s2_p1', + 'blocks': [block.config for block in self.blocks] + } + + def set_bn_param(self, momentum, eps): + for m in self.modules(): + if isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.BatchNorm1d): + m.momentum = momentum + m.eps = eps + return + + @staticmethod + def build_from_config(config): + first_conv_config = config['first_conv'] + match_obj = re.match(r'conv_in(\d+)_out(\d+)_k(\d+)_s(\d+)_p(\d+)', + first_conv_config) + in_channel = int(match_obj.group(1)) + out_channel = int(match_obj.group(2)) + kernel_size = int(match_obj.group(3)) + stride = int(match_obj.group(4)) + padding = int(match_obj.group(5)) + first_conv = nn.Sequential( + nn.Conv2d( + in_channel, + out_channel, + kernel_size, + stride, + padding, + bias=False), + nn.BatchNorm2d(out_channel), + nn.ReLU(inplace=True), + ) + blocks = [] + for block_config in config['blocks']: + blocks.append( + MobileInvertedResidualBlock.build_from_config(block_config)) + net = NasRecBackbone(first_conv, blocks) + if 'bn' in config: + net.set_bn_param(**config['bn']) + else: + net.set_bn_param(momentum=0.1, eps=1e-3) + return net + + +class CompactDetBackbone(NasRecBackbone): + ''' + proxyless nas backbone, 5M. + ''' + + def __init__(self, + width_stages, + input_channel=None, + bn_param=(0.1, 1e-3), + **kwargs): + + if input_channel is None: + input_channel = width_stages[0] + first_conv = nn.Sequential( + nn.Conv2d( + 3, + input_channel, + kernel_size=(3, 3), + stride=2, + padding=1, + bias=False), nn.BatchNorm2d(input_channel), + nn.ReLU(inplace=True)) + + conv_candidates = [ + '5x5_MBConv2', '5x5_MBConv4', '3x3_MBConv2', '3x3_MBConv4', + '13_MixConv2', '13_MixConv4', '35_MixConv2', '35_MixConv4', + '135_MixConv2', '135_MixConv4', '13_LinMixConv', '35_LinMixConv', + '135_LinMixConv', '13_RepConv2', '13_RepConv4', '35_RepConv2', + '35_RepConv4', '135_RepConv2', '135_RepConv4', 'Zero' + ] + + se_candidates = ['SE_2', 'SE_4', 'SE_8', 'Zero'] + + conv_op_ids = [ + 15, 17, 17, 17, 17, 0, 16, 16, 18, 18, 16, 2, 16, 18, 16, 18, 18, + 2, 1, 18, 18, 18, 16, 2 + ] + + n_cell_stages = [5, 5, 5, 5] + + stride_stages = [(2, 2), (2, 2), (2, 2), (2, 2)] + + if se_candidates: + assert len(conv_op_ids) == sum(n_cell_stages) + 4 + else: + assert len(conv_op_ids) == sum(n_cell_stages) + + blocks = [] + + for width, n_cell, s in zip(width_stages, n_cell_stages, + stride_stages): + for i in range(n_cell): + if i == 0: + stride = s + else: + stride = (1, 1) + block_i = len(blocks) + conv_op = conv_func_by_name( + conv_candidates[conv_op_ids[block_i]])(input_channel, + width, stride) + + if stride == (1, 1) and input_channel == width: + shortcut = IdentityLayer() + else: + shortcut = None + inverted_residual_block = MobileInvertedResidualBlock( + conv_op, shortcut) + blocks.append(inverted_residual_block) + input_channel = width + + if se_candidates is not None: + block_i = len(blocks) + se_op = conv_func_by_name(se_candidates[conv_op_ids[block_i]])( + input_channel, width, stride) + shortcut = IdentityLayer() + inverted_residual_block = MobileInvertedResidualBlock( + se_op, shortcut) + blocks.append(inverted_residual_block) + + self.out_channel = input_channel + + super(CompactDetBackbone, self).__init__(first_conv, blocks) + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) diff --git a/tests/pipelines/test_ocr_detection.py b/tests/pipelines/test_ocr_detection.py index 0ed2e59c..f5fc0c63 100644 --- a/tests/pipelines/test_ocr_detection.py +++ b/tests/pipelines/test_ocr_detection.py @@ -13,6 +13,7 @@ class OCRDetectionTest(unittest.TestCase): self.model_id = 'damo/cv_resnet18_ocr-detection-line-level_damo' self.model_id_vlpt = 'damo/cv_resnet50_ocr-detection-vlpt' self.model_id_db = 'damo/cv_resnet18_ocr-detection-db-line-level_damo' + self.model_id_db_nas = 'damo/cv_proxylessnas_ocr-detection-db-line-level_damo' self.test_image = 'data/test/images/ocr_detection.jpg' self.test_image_vlpt = 'data/test/images/ocr_detection_vlpt.jpg' self.task = Tasks.ocr_detection @@ -37,6 +38,15 @@ class OCRDetectionTest(unittest.TestCase): ocr_detection = pipeline(Tasks.ocr_detection, model=self.model_id_db) self.pipeline_inference(ocr_detection, self.test_image) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_dbnas_with_model_from_modelhub(self): + ocr_detection = pipeline( + Tasks.ocr_detection, + model=self.model_id_db_nas, + model_revision='v1.0.0', + ) + self.pipeline_inference(ocr_detection, self.test_image) + @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_run_modelhub_default_model(self): ocr_detection = pipeline(Tasks.ocr_detection) From 73f8d3989799d027bc77ba8f77ca3078d801e410 Mon Sep 17 00:00:00 2001 From: pangda Date: Mon, 5 Jun 2023 15:17:45 +0800 Subject: [PATCH 61/76] Add ner example for flextrain Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12835095 * update named_entity_recognition example --- .../finetune_named_entity_recognition.py | 97 +++++++++++++++++++ .../named_entity_recognition/run_train.sh | 26 +++++ 2 files changed, 123 insertions(+) create mode 100644 examples/pytorch/named_entity_recognition/finetune_named_entity_recognition.py create mode 100644 examples/pytorch/named_entity_recognition/run_train.sh diff --git a/examples/pytorch/named_entity_recognition/finetune_named_entity_recognition.py b/examples/pytorch/named_entity_recognition/finetune_named_entity_recognition.py new file mode 100644 index 00000000..67a091f7 --- /dev/null +++ b/examples/pytorch/named_entity_recognition/finetune_named_entity_recognition.py @@ -0,0 +1,97 @@ +import os +from dataclasses import dataclass, field + +from adaseq.data.data_collators.base import build_data_collator +from adaseq.data.dataset_manager import DatasetManager +from adaseq.data.preprocessors.nlp_preprocessor import build_preprocessor +from adaseq.training.default_trainer import DefaultTrainer as AdaSeqTrainer + +from modelscope import MsDataset, TrainingArgs, build_dataset_from_file + + +@dataclass(init=False) +class NamedEntityRecognitionArguments(TrainingArgs): + preprocessor: str = field( + default='sequence-labeling-preprocessor', + metadata={ + 'help': 'The preprocessor type', + 'cfg_node': 'preprocessor.type' + }) + + sequence_length: int = field( + default=150, + metadata={ + 'cfg_node': 'preprocessor.max_length', + 'help': 'The parameters for train dataset', + }) + + data_collator: str = field( + default='SequenceLabelingDataCollatorWithPadding', + metadata={ + 'cfg_node': 'data_collator', + 'help': 'The type of data collator', + }) + + dropout: float = field( + default=0.0, + metadata={ + 'cfg_node': 'model.dropout', + 'help': 'Dropout rate', + }) + + use_crf: bool = field( + default=True, + metadata={ + 'cfg_node': 'model.use_crf', + 'help': 'Whether to add a CRF decoder layer', + }) + + crf_lr: float = field( + default=5.0e-1, metadata={ + 'help': 'Learning rate for CRF layer', + }) + + +training_args = NamedEntityRecognitionArguments().parse_cli() +config, args = training_args.to_config() +print(args) + +if args.dataset_json_file is None: + train_dataset = MsDataset.load( + args.train_dataset_name, + subset_name=args.train_subset_name, + split=args.train_split, + namespace=args.train_dataset_namespace).to_hf_dataset() + validation_dataset = MsDataset.load( + args.val_dataset_name, + subset_name=args.val_subset_name, + split=args.val_split, + namespace=args.val_dataset_namespace).to_hf_dataset() +else: + train_dataset, validation_dataset = build_dataset_from_file( + args.dataset_json_file) +dm = DatasetManager({ + 'train': train_dataset, + 'valid': validation_dataset +}, labels={'type': 'count_span_labels'}) # yapf: disable + +config.preprocessor.model_dir = args.model +config.model.embedder = {'model_name_or_path': args.model} +preprocessor = build_preprocessor(config.preprocessor, labels=dm.labels) +config.model.id_to_label = preprocessor.id_to_label +data_collator = build_data_collator(preprocessor.tokenizer, + dict(type=config.data_collator)) +config.train.optimizer.param_groups = [{'regex': 'crf', 'lr': args.crf_lr}] + +cfg_file = os.path.join(config.train.work_dir, 'config.yaml') +config.dump(cfg_file) + +kwargs = dict( + cfg_file=cfg_file, + work_dir=config.train.work_dir, + dataset_manager=dm, + data_collator=data_collator, + preprocessor=preprocessor) + +trainer = AdaSeqTrainer(**kwargs) +trainer.train() diff --git a/examples/pytorch/named_entity_recognition/run_train.sh b/examples/pytorch/named_entity_recognition/run_train.sh new file mode 100644 index 00000000..91b2c5bf --- /dev/null +++ b/examples/pytorch/named_entity_recognition/run_train.sh @@ -0,0 +1,26 @@ +PYTHONPATH=. python examples/pytorch/named_entity_recognition/finetune_named_entity_recognition.py \ + --task 'named-entity-recognition' \ + --work_dir './tmp' \ + --model_type 'sequence-labeling-model' \ + --model 'damo/nlp_structbert_backbone_base_std' \ + --dropout 0.1 \ + --use_crf true \ + --train_dataset_name 'resume_ner' \ + --train_dataset_namespace 'damo' \ + --train_split 'train' \ + --val_dataset_name 'resume_ner' \ + --val_dataset_namespace 'damo' \ + --val_split 'dev' \ + --preprocessor 'sequence-labeling-preprocessor' \ + --sequence_length 150 \ + --data_collator 'SequenceLabelingDataCollatorWithPadding' \ + --max_epochs 5 \ + --per_device_train_batch_size 16 \ + --train_data_worker 0 \ + --eval_data_worker 0 \ + --lr 5.0e-5 \ + --lr_scheduler LinearLR \ + --lr_scheduler_params 'start_factor=1.0,end_factor=0.0,total_iters=5' \ + --eval_metrics ner-metric \ + --save_best_checkpoint true \ + --metric_for_best_model f1 \ From 25a7398aeab1a36466fd5e5278317f7de9032d19 Mon Sep 17 00:00:00 2001 From: "mulin.lyh" Date: Thu, 8 Jun 2023 10:01:58 +0800 Subject: [PATCH 62/76] docker file py38 and py37 compatible merge Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12722824 * debug * add missing deps * add stanza * fix numpy issue * numpy version * tf case * numpy <= 1.22.0 * fix tf 1.15.5 case * add python38 and python37 compatible docker file * cv add fvcore * add easycv * remove debug code * fix style issue * fix python3.7 python3.8 compatible issue * comment * modify run config * scipy<=1.7.3 * revert scipy<=1.7.3 * fix compatible bug * fix compatible bug * add easycv * update version: 1.6.1rc0 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12754942 fix python3.8 and python3.8 compatible issue * numpy version change * numpy version * numpy version * fix numpy version * fix numpy version * update version: 1.6.1rc0 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12754942 fix python3.8 and python3.8 compatible issue * restore setup.py * restore setup.py * add build base image * fix style * numpy version fix * optimize build image, split base and modelscope image * fix style issue * modify example path * modify examples folder * add SETUPTOOLS_USE_DISTUTILS=stdlib comments --- .dev_scripts/build_base_image.sh | 119 ++++++++++++++++++++++++ .dev_scripts/build_image.sh | 24 +++-- docker/Dockerfile.ubuntu | 122 ++++--------------------- docker/Dockerfile.ubuntu_base | 136 ++++++++++++++++++++++++++++ docker/rcfiles/sources.list.aliyun | 31 ++----- docker/scripts/install_apex.sh | 3 +- requirements/audio/audio_kws.txt | 1 - requirements/audio/audio_signal.txt | 1 - requirements/cv.txt | 6 +- requirements/framework.txt | 4 +- requirements/nlp.txt | 2 + requirements/tensorflow1x.txt | 2 +- tests/run_config.yaml | 35 +------ 13 files changed, 317 insertions(+), 169 deletions(-) create mode 100644 .dev_scripts/build_base_image.sh create mode 100644 docker/Dockerfile.ubuntu_base diff --git a/.dev_scripts/build_base_image.sh b/.dev_scripts/build_base_image.sh new file mode 100644 index 00000000..e60f6b3c --- /dev/null +++ b/.dev_scripts/build_base_image.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# default values. +BASE_CPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu:20.04 +BASE_GPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu:20.04-cuda11.3.0-cudnn8-devel +MODELSCOPE_REPO_ADDRESS=reg.docker.alibaba-inc.com/modelscope/modelscope +python_version=3.7.13 +torch_version=1.11.0 +cudatoolkit_version=11.3 +tensorflow_version=1.15.5 +version=None +is_cpu=False +function usage(){ + echo "usage: build.sh " + echo " --python=python_version set python version, default: $python_version" + echo " --torch=torch_version set pytorch version, fefault: $torch_version" + echo " --tensorflow=tensorflow_version set tensorflow version, default: $tensorflow_version" + echo " --version=version set image version, default: $version" + echo " --test option for run test before push image, only push on ci test pass" + echo " --cpu option for build cpu version" + echo " --dsw option for build dsw version" + echo " --ci option for build ci version" + echo " --push option for push image to remote repo" +} +for i in "$@"; do + case $i in + --python=*) + python_version="${i#*=}" + shift + ;; + --torch=*) + torch_version="${i#*=}" + shift # pytorch version + ;; + --tensorflow=*) + tensorflow_version="${i#*=}" + shift # tensorflow version + ;; + --version=*) + version="${i#*=}" + shift # version + ;; + --cpu) + is_cpu=True + shift # is cpu image + ;; + --push) + is_push=True + shift # option for push image to remote repo + ;; + --help) + usage + exit 0 + ;; + -*|--*) + echo "Unknown option $i" + usage + exit 1 + ;; + *) + ;; + esac +done + +if [ "$version" == "None" ]; then + echo "version must specify!" + exit 1 +fi +if [ "$is_cpu" == "True" ]; then + export BASE_IMAGE=$BASE_CPU_IMAGE + base_tag=ubuntu20.04 + export USE_GPU=False +else + export BASE_IMAGE=$BASE_GPU_IMAGE + base_tag=ubuntu20.04-cuda11.3.0 + export USE_GPU=True +fi +if [[ $python_version == 3.7* ]]; then + base_tag=$base_tag-py37 +elif [[ $python_version == 3.8* ]]; then + base_tag=$base_tag-py38 +elif [[ $python_version == 3.9* ]]; then + base_tag=$base_tag-py39 +else + echo "Unsupport python version: $python_version" + exit 1 +fi + +target_image_tag=$base_tag-torch$torch_version-tf$tensorflow_version-base-$version +export IMAGE_TO_BUILD=$MODELSCOPE_REPO_ADDRESS:$target_image_tag +export PYTHON_VERSION=$python_version +export TORCH_VERSION=$torch_version +export CUDATOOLKIT_VERSION=$cudatoolkit_version +export TENSORFLOW_VERSION=$tensorflow_version +echo -e "Building image with:\npython$python_version\npytorch$torch_version\ntensorflow:$tensorflow_version\ncudatoolkit:$cudatoolkit_version\ncpu:$is_cpu\n" +docker_file_content=`cat docker/Dockerfile.ubuntu_base` +printf "$docker_file_content" > Dockerfile + +while true +do + docker build -t $IMAGE_TO_BUILD \ + --build-arg USE_GPU \ + --build-arg BASE_IMAGE \ + --build-arg PYTHON_VERSION \ + --build-arg TORCH_VERSION \ + --build-arg CUDATOOLKIT_VERSION \ + --build-arg TENSORFLOW_VERSION \ + -f Dockerfile . + if [ $? -eq 0 ]; then + echo "Image build done" + break + else + echo "Running docker build command error, we will retry" + fi +done + +if [ "$is_push" == "True" ]; then + echo "Pushing image: $IMAGE_TO_BUILD" + docker push $IMAGE_TO_BUILD +fi diff --git a/.dev_scripts/build_image.sh b/.dev_scripts/build_image.sh index 6bc8b5e4..948ecb64 100644 --- a/.dev_scripts/build_image.sh +++ b/.dev_scripts/build_image.sh @@ -1,7 +1,9 @@ #!/bin/bash # default values. -BASE_CPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu:20.04 -BASE_GPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu:20.04-cuda11.3.0-cudnn8-devel +BASE_PY38_CPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-py38-torch1.11.0-tf1.15.5-base-1.6.1 +BASE_PY38_GPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-cuda11.3.0-py38-torch1.11.0-tf1.15.5-base-1.6.1 +BASE_PY37_CPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-py37-torch1.11.0-tf1.15.5-base-1.6.1 +BASE_PY37_GPU_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-cuda11.3.0-py37-torch1.11.0-tf1.15.5-base-1.6.1 MODELSCOPE_REPO_ADDRESS=reg.docker.alibaba-inc.com/modelscope/modelscope python_version=3.7.13 torch_version=1.11.0 @@ -86,20 +88,30 @@ if [ "$modelscope_version" == "None" ]; then exit 1 fi if [ "$is_cpu" == "True" ]; then - export BASE_IMAGE=$BASE_CPU_IMAGE base_tag=ubuntu20.04 export USE_GPU=False else - export BASE_IMAGE=$BASE_GPU_IMAGE base_tag=ubuntu20.04-cuda11.3.0 export USE_GPU=True fi if [[ $python_version == 3.7* ]]; then + if [ "$is_cpu" == "True" ]; then + echo "Building python3.7 cpu image" + export BASE_IMAGE=$BASE_PY37_CPU_IMAGE + else + echo "Building python3.7 gpu image" + export BASE_IMAGE=$BASE_PY37_GPU_IMAGE + fi base_tag=$base_tag-py37 elif [[ $python_version == 3.8* ]]; then + if [ "$is_cpu" == "True" ]; then + echo "Building python3.8 cpu image" + export BASE_IMAGE=$BASE_PY38_CPU_IMAGE + else + echo "Building python3.8 gpu image" + export BASE_IMAGE=$BASE_PY38_GPU_IMAGE + fi base_tag=$base_tag-py38 -elif [[ $python_version == 3.9* ]]; then - base_tag=$base_tag-py39 else echo "Unsupport python version: $python_version" exit 1 diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index 27c0f1f3..2579a6de 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -1,102 +1,5 @@ -ARG BASE_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu:20.04-cuda11.3.0-cudnn8-devel +ARG BASE_IMAGE=reg.docker.alibaba-inc.com/modelscope/modelscope:ubuntu20.04-cuda11.3.0-py37-torch1.11.0-tf1.15.5-1.6.1 FROM $BASE_IMAGE -ARG DEBIAN_FRONTEND=noninteractive -ENV TZ=Asia/Shanghai -ENV CONDA_DIR /opt/conda -ENV PATH="${CONDA_DIR}/bin:${PATH}" -ENV arch=x86_64 -SHELL ["/bin/bash", "-c"] -COPY docker/rcfiles /tmp/resources -COPY docker/jupyter_plugins /tmp/resources/jupyter_plugins -RUN apt-get update && apt-get install -y --reinstall ca-certificates && \ - apt-get clean && \ - cp /tmp/resources/ubuntu20.04_sources.tuna /etc/apt/sources.list && \ - apt-get update && \ - apt-get install -y locales wget git strace gdb sox libopenmpi-dev curl strace vim ffmpeg libsm6 tzdata language-pack-zh-hans ttf-wqy-microhei ttf-wqy-zenhei xfonts-wqy libxext6 build-essential ninja-build && \ - wget https://packagecloud.io/github/git-lfs/packages/debian/bullseye/git-lfs_3.2.0_amd64.deb/download -O ./git-lfs_3.2.0_amd64.deb && \ - dpkg -i ./git-lfs_3.2.0_amd64.deb && \ - rm -f ./git-lfs_3.2.0_amd64.deb && \ - locale-gen zh_CN && \ - locale-gen zh_CN.utf8 && \ - update-locale LANG=zh_CN.UTF-8 LC_ALL=zh_CN.UTF-8 LANGUAGE=zh_CN.UTF-8 && \ - ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ - dpkg-reconfigure --frontend noninteractive tzdata && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -ENV LANG=zh_CN.UTF-8 LANGUAGE=zh_CN.UTF-8 LC_ALL=zh_CN.UTF-8 - -#install and config python -ARG PYTHON_VERSION=3.7.13 -RUN wget --quiet https://mirrors.aliyun.com/anaconda/miniconda/Miniconda3-latest-Linux-${arch}.sh -O ./miniconda.sh && \ - /bin/bash miniconda.sh -b -p /opt/conda && \ - rm -f miniconda.sh && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ - cp /tmp/resources/conda.tuna ~/.condarc && \ - source /root/.bashrc && \ - conda install --yes python==${PYTHON_VERSION} && \ - pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \ - pip config set install.trusted-host mirrors.aliyun.com - -ARG USE_GPU=True - -# install pytorch -ARG TORCH_VERSION=1.12.0 -ARG CUDATOOLKIT_VERSION=11.3 -RUN if [ "$USE_GPU" = "True" ] ; then \ - pip install --no-cache-dir torch==$TORCH_VERSION torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113; \ - else \ - pip install --no-cache-dir torch==$TORCH_VERSION torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu; \ - fi - -# install tensorflow -ARG TENSORFLOW_VERSION=1.15.5 -RUN if [ "$USE_GPU" = "True" ] ; then \ - pip install --no-cache-dir tensorflow==$TENSORFLOW_VERSION -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html; \ - else \ - pip install --no-cache-dir tensorflow==$TENSORFLOW_VERSION; \ - fi - -# mmcv-full<=1.7.0 for mmdet3d compatible -RUN if [ "$USE_GPU" = "True" ] ; then \ - CUDA_HOME=/usr/local/cuda TORCH_CUDA_ARCH_LIST="5.0 5.2 6.0 6.1 7.0 7.5 8.0 8.6" MMCV_WITH_OPS=1 MAX_JOBS=8 FORCE_CUDA=1 pip install --no-cache-dir 'mmcv-full<=1.7.0' && pip cache purge; \ - else \ - MMCV_WITH_OPS=1 MAX_JOBS=8 pip install --no-cache-dir 'mmcv-full<=1.7.0' && pip cache purge; \ - fi - -# default shell bash -ENV SHELL=/bin/bash -# install special package -RUN if [ "$USE_GPU" = "True" ] ; then \ - pip install --no-cache-dir dgl-cu113 dglgo -f https://data.dgl.ai/wheels/repo.html; \ - else \ - pip install --no-cache-dir dgl dglgo -f https://data.dgl.ai/wheels/repo.html; \ - fi - -# copy install scripts -COPY docker/scripts/install_unifold.sh docker/scripts/install_colmap.sh docker/scripts/install_pytorch3d_nvdiffrast.sh docker/scripts/install_tiny_cuda_nn.sh docker/scripts/install_apex.sh /tmp/ - -# for uniford -RUN if [ "$USE_GPU" = "True" ] ; then \ - bash /tmp/install_unifold.sh; \ - else \ - echo 'cpu unsupport uniford'; \ - fi - -RUN if [ "$USE_GPU" = "True" ] ; then \ - pip install --no-cache-dir git+https://github.com/gxd1994/Pointnet2.PyTorch.git@master#subdirectory=pointnet2; \ - else \ - echo 'cpu unsupport Pointnet2'; \ - fi - -RUN pip install --no-cache-dir detectron2==0.3 -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html - -# 3d supports -RUN bash /tmp/install_colmap.sh -RUN bash /tmp/install_tiny_cuda_nn.sh -RUN bash /tmp/install_pytorch3d_nvdiffrast.sh -# end of 3D # install modelscope COPY requirements /var/modelscope @@ -115,12 +18,25 @@ RUN mkdir -p /root/.local/share/jupyter/labextensions/ && \ cp -r /tmp/resources/jupyter_plugins/* /root/.local/share/jupyter/labextensions/ COPY docker/scripts/modelscope_env_init.sh /usr/local/bin/ms_env_init.sh -RUN pip install --no-cache-dir xtcocotools==1.12 detectron2==0.3 -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --force +# python3.8 pip install git+https://github.com/jin-s13/xtcocoapi.git@v1.13 +# pip install git+https://github.com/gatagat/lap.git@v0.4.0 +RUN pip install --no-cache-dir text2sql_lgesql==1.3.0 \ + git+https://github.com/jin-s13/xtcocoapi.git@v1.13 \ + git+https://github.com/gatagat/lap.git@v0.4.0 \ + detectron2==0.3 -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --force --no-deps -# speechbrain==0.5.7 for audio compatible -RUN pip install --no-cache-dir speechbrain==0.5.7 adaseq>=0.5.0 mmcls>=0.21.0 mmdet>=2.25.0 decord>=0.6.0 numpy==1.18.5 wenetruntime==1.11.0 ipykernel fairseq fasttext deepspeed +RUN pip install --no-cache-dir mpi4py paint_ldm adaseq>=0.5.0 \ + mmcls>=0.21.0 mmdet>=2.25.0 decord>=0.6.0 wenetruntime==1.11.0 \ + ipykernel fairseq fasttext deepspeed -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html + +# for cpu install cpu version faiss, faiss depends on blas lib, we install libopenblas TODO rename gpu or cpu version faiss RUN if [ "$USE_GPU" = "True" ] ; then \ - bash /tmp/install_apex.sh; \ + pip install --no-cache-dir funtextprocessing kwsbp==0.0.6 faiss==1.7.2 safetensors typeguard==2.13.3 scikit-learn 'pandas<1.4.0' pai-easycv librosa==0.9.2 funasr -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html; \ else \ - echo 'cpu unsupport apex'; \ + pip install --no-cache-dir funtextprocessing kwsbp==0.0.6 https://modelscope.oss-cn-beijing.aliyuncs.com/releases/dependencies/faiss-1.7.2-py37-none-linux_x86_64.whl safetensors typeguard==2.13.3 scikit-learn 'pandas<1.4.0' pai-easycv librosa==0.9.2 funasr -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html; \ fi + +COPY examples /modelscope/examples + +# for pai-easycv setup compatiblity issue +ENV SETUPTOOLS_USE_DISTUTILS=stdlib diff --git a/docker/Dockerfile.ubuntu_base b/docker/Dockerfile.ubuntu_base new file mode 100644 index 00000000..4d92ceda --- /dev/null +++ b/docker/Dockerfile.ubuntu_base @@ -0,0 +1,136 @@ +ARG BASE_IMAGE=reg.docker.alibaba-inc.com/modelscope/ubuntu:20.04-cuda11.3.0-cudnn8-devel +FROM $BASE_IMAGE +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=Asia/Shanghai +ENV CONDA_DIR /opt/conda +ENV PATH="${CONDA_DIR}/bin:${PATH}" +ENV arch=x86_64 +SHELL ["/bin/bash", "-c"] +COPY docker/rcfiles /tmp/resources +COPY docker/jupyter_plugins /tmp/resources/jupyter_plugins +RUN apt-get update && apt-get install -y --reinstall ca-certificates && \ + apt-get clean && \ + cp /tmp/resources/sources.list.aliyun /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y locales wget git strace gdb sox libopenmpi-dev curl \ + libgeos-dev strace vim ffmpeg libsm6 tzdata language-pack-zh-hans \ + ttf-wqy-microhei ttf-wqy-zenhei xfonts-wqy libxext6 build-essential ninja-build && \ + wget https://packagecloud.io/github/git-lfs/packages/debian/bullseye/git-lfs_3.2.0_amd64.deb/download -O ./git-lfs_3.2.0_amd64.deb && \ + dpkg -i ./git-lfs_3.2.0_amd64.deb && \ + rm -f ./git-lfs_3.2.0_amd64.deb && \ + locale-gen zh_CN && \ + locale-gen zh_CN.utf8 && \ + update-locale LANG=zh_CN.UTF-8 LC_ALL=zh_CN.UTF-8 LANGUAGE=zh_CN.UTF-8 && \ + ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + dpkg-reconfigure --frontend noninteractive tzdata && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV LANG=zh_CN.UTF-8 LANGUAGE=zh_CN.UTF-8 LC_ALL=zh_CN.UTF-8 + +#install and config python +ARG PYTHON_VERSION=3.7.13 +# Miniconda3-py37_23.1.0-1-Linux-x86_64.sh is last python3.7 version +RUN if [ "$PYTHON_VERSION" = "3.7.13" ] ; then \ + wget --quiet https://mirrors.aliyun.com/anaconda/miniconda/Miniconda3-py37_23.1.0-1-Linux-x86_64.sh -O ./miniconda.sh && \ + /bin/bash miniconda.sh -b -p /opt/conda && \ + rm -f miniconda.sh && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + cp /tmp/resources/conda.tuna ~/.condarc && \ + source /root/.bashrc && \ + conda install --yes python==${PYTHON_VERSION} && \ + pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \ + pip config set install.trusted-host mirrors.aliyun.com;\ +else \ + wget --quiet https://mirrors.aliyun.com/anaconda/miniconda/Miniconda3-latest-Linux-${arch}.sh -O ./miniconda.sh && \ + /bin/bash miniconda.sh -b -p /opt/conda && \ + rm -f miniconda.sh && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + cp /tmp/resources/conda.tuna ~/.condarc && \ + source /root/.bashrc && \ + conda install --yes python==${PYTHON_VERSION} && \ + pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \ + pip config set install.trusted-host mirrors.aliyun.com;\ +fi + +ARG USE_GPU=True + +# install pytorch +ARG TORCH_VERSION=1.12.0 +ARG CUDATOOLKIT_VERSION=cu113 +RUN if [ "$USE_GPU" = "True" ] ; then \ + pip install --no-cache-dir torch==$TORCH_VERSION torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113; \ + else \ + pip install --no-cache-dir torch==$TORCH_VERSION torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu; \ + fi + +# install tensorflow +ARG TENSORFLOW_VERSION=1.15.5 +RUN if [ "$USE_GPU" = "True" ] ; then \ + pip install --no-cache-dir tensorflow==$TENSORFLOW_VERSION -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html; \ + else \ + # only python 3.7 has tensorflow 1.15.5 + if [ "$PYTHON_VERSION" = "3.7.13" ] ; then \ + pip install --no-cache-dir tensorflow==$TENSORFLOW_VERSION; \ + else \ + pip install --no-cache-dir numpy==1.18.5 https://modelscope.oss-cn-beijing.aliyuncs.com/releases/dependencies/tensorflow-1.15.5-cp38-cp38-linux_x86_64.whl; \ + fi \ + fi + +# mmcv-full<=1.7.0 for mmdet3d compatible +RUN if [ "$USE_GPU" = "True" ] ; then \ + CUDA_HOME=/usr/local/cuda TORCH_CUDA_ARCH_LIST="5.0 5.2 6.0 6.1 7.0 7.5 8.0 8.6" MMCV_WITH_OPS=1 MAX_JOBS=8 FORCE_CUDA=1 pip install --no-cache-dir 'mmcv-full<=1.7.0' && pip cache purge; \ + else \ + MMCV_WITH_OPS=1 MAX_JOBS=8 pip install --no-cache-dir 'mmcv-full<=1.7.0' && pip cache purge; \ + fi + +# default shell bash +ENV SHELL=/bin/bash +# install special package +RUN if [ "$USE_GPU" = "True" ] ; then \ + pip install --no-cache-dir dgl-cu113 dglgo -f https://data.dgl.ai/wheels/repo.html; \ + else \ + pip install --no-cache-dir dgl==0.9.0 dglgo -f https://data.dgl.ai/wheels/repo.html; \ + fi + +# copy install scripts +COPY docker/scripts/install_unifold.sh docker/scripts/install_colmap.sh docker/scripts/install_pytorch3d_nvdiffrast.sh docker/scripts/install_tiny_cuda_nn.sh docker/scripts/install_apex.sh /tmp/ + +# for uniford +RUN if [ "$USE_GPU" = "True" ] ; then \ + bash /tmp/install_unifold.sh; \ + else \ + echo 'cpu unsupport uniford'; \ + fi + +RUN if [ "$USE_GPU" = "True" ] ; then \ + pip install --no-cache-dir git+https://github.com/gxd1994/Pointnet2.PyTorch.git@master#subdirectory=pointnet2; \ + else \ + echo 'cpu unsupport Pointnet2'; \ + fi + +# 3d supports +RUN if [ "$USE_GPU" = "True" ] ; then \ + bash /tmp/install_colmap.sh; \ + else \ + echo 'cpu unsupport colmap'; \ + fi +RUN if [ "$USE_GPU" = "True" ] ; then \ + bash /tmp/install_tiny_cuda_nn.sh \ + else \ + echo 'cpu unsupport tiny_cudann'; \ + fi +RUN if [ "$USE_GPU" = "True" ] ; then \ + bash /tmp/install_pytorch3d_nvdiffrast.sh; \ + else \ + echo 'cpu unsupport pytorch3d nvdiffrast'; \ + fi +# end of 3D +# install apex after deepspeed +RUN if [ "$USE_GPU" = "True" ] ; then \ + bash /tmp/install_apex.sh; \ + else \ + echo 'cpu unsupport apex'; \ + fi diff --git a/docker/rcfiles/sources.list.aliyun b/docker/rcfiles/sources.list.aliyun index 120bb1f1..1ebf4ae5 100644 --- a/docker/rcfiles/sources.list.aliyun +++ b/docker/rcfiles/sources.list.aliyun @@ -1,25 +1,14 @@ -deb http://mirrors.aliyun.com/ubuntu/ bionic main restricted -# deb-src http://mirrors.aliyun.com/ubuntu/ bionic main restricted +deb https://mirrors.aliyun.com/ubuntu/ focal main restricted universe multiverse +# deb-src https://mirrors.aliyun.com/ubuntu/ focal main restricted universe multiverse -deb http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted -# deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted +deb https://mirrors.aliyun.com/ubuntu/ focal-security main restricted universe multiverse +# deb-src https://mirrors.aliyun.com/ubuntu/ focal-security main restricted universe multiverse -deb http://mirrors.aliyun.com/ubuntu/ bionic universe -# deb-src http://mirrors.aliyun.com/ubuntu/ bionic universe -deb http://mirrors.aliyun.com/ubuntu/ bionic-updates universe -# deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates universe +deb https://mirrors.aliyun.com/ubuntu/ focal-updates main restricted universe multiverse +# deb-src https://mirrors.aliyun.com/ubuntu/ focal-updates main restricted universe multiverse -deb http://mirrors.aliyun.com/ubuntu/ bionic multiverse -# deb-src http://mirrors.aliyun.com/ubuntu/ bionic multiverse -deb http://mirrors.aliyun.com/ubuntu/ bionic-updates multiverse -# deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates multiverse +# deb https://mirrors.aliyun.com/ubuntu/ focal-proposed main restricted universe multiverse +# deb-src https://mirrors.aliyun.com/ubuntu/ focal-proposed main restricted universe multiverse -deb http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse -# deb-src http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse - -deb http://mirrors.aliyun.com/ubuntu bionic-security main restricted -# deb-src http://mirrors.aliyun.com/ubuntu bionic-security main restricted -deb http://mirrors.aliyun.com/ubuntu bionic-security universe -# deb-src http://mirrors.aliyun.com/ubuntu bionic-security universe -deb http://mirrors.aliyun.com/ubuntu bionic-security multiverse -# deb-src http://mirrors.aliyun.com/ubuntu bionic-security multiverse +deb https://mirrors.aliyun.com/ubuntu/ focal-backports main restricted universe multiverse +# deb-src https://mirrors.aliyun.com/ubuntu/ focal-backports main restricted universe multiverse diff --git a/docker/scripts/install_apex.sh b/docker/scripts/install_apex.sh index 47f34da7..40d9f268 100644 --- a/docker/scripts/install_apex.sh +++ b/docker/scripts/install_apex.sh @@ -1,6 +1,7 @@ export MAX_JOBS=16 \ && git clone https://github.com/NVIDIA/apex \ && cd apex \ -&& TORCH_CUDA_ARCH_LIST="6.0;6.1;6.2;7.0;7.5;8.0;8.6" pip install -v --disable-pip-version-check --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./ \ +&& git checkout 6bd01c4b99a84648ad5e5238a959735e6936c813 \ +&& TORCH_CUDA_ARCH_LIST="6.0;6.1;6.2;7.0;7.5;8.0;8.6" pip install -v --disable-pip-version-check --no-cache --global-option="--cpp_ext" --global-option="--cuda_ext" ./ \ && cd .. \ && rm -rf apex diff --git a/requirements/audio/audio_kws.txt b/requirements/audio/audio_kws.txt index 4118f3ed..276a0a2f 100644 --- a/requirements/audio/audio_kws.txt +++ b/requirements/audio/audio_kws.txt @@ -1,7 +1,6 @@ kaldiio kwsbp>=0.0.6 matplotlib -numpy py_sound_connect>=0.1 scipy SoundFile>0.10 diff --git a/requirements/audio/audio_signal.txt b/requirements/audio/audio_signal.txt index 16a18e67..62944b71 100644 --- a/requirements/audio/audio_signal.txt +++ b/requirements/audio/audio_signal.txt @@ -2,7 +2,6 @@ hyperpyyaml librosa==0.9.2 MinDAEC mir_eval>=0.7 -numpy rotary_embedding_torch>=0.1.5 scipy SoundFile>0.10 diff --git a/requirements/cv.txt b/requirements/cv.txt index 0cec3659..c48d82e2 100644 --- a/requirements/cv.txt +++ b/requirements/cv.txt @@ -16,6 +16,7 @@ fastai>=1.0.51 ffmpeg>=1.4 ffmpeg-python>=0.2.0 ftfy +fvcore imageio>=2.9.0 imageio-ffmpeg>=0.4.2 imgaug>=0.4.0 @@ -44,16 +45,19 @@ pandas panopticapi plyfile>=0.7.4 psutil +pyclipper PyMCubes pytorch-lightning regex -scikit-image>=0.19.3 +# <0.20.0 for compatible python3.7 python3.8 +scikit-image>=0.19.3,<0.20.0 scikit-learn>=0.20.1 shapely shotdetect_scenedetect_lgss>=0.0.4 smplx tensorflow-estimator>=1.15.1 tf_slim +thop timm>=0.4.9 torchmetrics>=0.6.2 torchsummary>=1.5.1 diff --git a/requirements/framework.txt b/requirements/framework.txt index e763ae63..3ba659fc 100644 --- a/requirements/framework.txt +++ b/requirements/framework.txt @@ -7,8 +7,8 @@ gast>=0.2.2 # for python3.7 python3.8 compatible numpy<=1.22.0 oss2 -# for datasets compatible -pandas<=1.5.3 +# for datasets compatible and py37 py38 compatible +pandas<1.4.0 Pillow>=6.2.0 # pyarrow 9.0.0 introduced event_loop core dump pyarrow>=6.0.0,!=9.0.0 diff --git a/requirements/nlp.txt b/requirements/nlp.txt index 2e22ef8f..18f9a08f 100644 --- a/requirements/nlp.txt +++ b/requirements/nlp.txt @@ -1,4 +1,5 @@ boto3 +embeddings en_core_web_sm>=2.3.5 filelock ftfy @@ -18,6 +19,7 @@ scikit_learn sentencepiece seqeval spacy>=2.3.5 +stanza subword_nmt>=0.3.8 termcolor tokenizers diff --git a/requirements/tensorflow1x.txt b/requirements/tensorflow1x.txt index b139efe1..5d680652 100644 --- a/requirements/tensorflow1x.txt +++ b/requirements/tensorflow1x.txt @@ -1 +1 @@ -numpy==1.18.5 +numpy<1.20.0 diff --git a/tests/run_config.yaml b/tests/run_config.yaml index ba678468..cfa08e4a 100644 --- a/tests/run_config.yaml +++ b/tests/run_config.yaml @@ -60,48 +60,19 @@ isolated: # test cases that may require excessive anmount of GPU memory or run - test_video_deinterlace.py - test_image_inpainting_sdv2.py - test_bad_image_detecting.py - - test_image_portrait_stylization_trainer.py - test_controllable_image_generation.py - test_image_colorization_trainer.py envs: default: # default env, case not in other env will in default, pytorch. dependencies: # requirement packages，pip install before test case run. - - numpy>=1.20,<=1.21.0 + - numpy>=1.20,<=1.22.0 - protobuf<4,>=3.20.2 + tensorflow1x: # cases excuted tensorflow1.x framework. requirements: # requirements files run before test case run. - tensorflow1x.txt dependencies: # requirement packages，pip install before test case run. - - numpy==1.18.5 + - numpy<1.20.0 tests: - - test_text_to_speech.py - - test_csanmt_translation.py - - test_translation_trainer.py - - test_translation_evaluation_trainer.py - - test_ocr_detection.py - - test_automatic_speech_recognition.py - - test_image_matting.py - - test_person_image_cartoon.py - - test_skin_retouching.py - - test_image_style_transfer.py - test_image_portrait_stylization_trainer.py - - test_language_identification.py - - test_language_guided_video_summarization_trainer.py - - test_motion_generation.py - - test_universal_matting.py - - test_dialog_modeling.py - - test_trainer.py - - test_abnormal_object_detection.py - - test_image_face_fusion.py - - test_ocr_detection_db_trainer.py - - test_language_guided_video_summarization.py - - test_interactive_translation_pipeline.py - - test_image_defrcn_fewshot_trainer.py - - test_automatic_post_editing.py - - test_human_reconstruction.py - - test_nerf_recon_acc_trainer.py - - test_nerf_recon_acc.py - - test_speech_signal_process.py - - test_tensorboard_hook.py - - test_efficient_diffusion_tuning_trainer.py From 75c90d8a980d12cb1f7f3bcb4b99d466813c756f Mon Sep 17 00:00:00 2001 From: zsl01670416 Date: Thu, 8 Jun 2023 19:36:05 +0800 Subject: [PATCH 63/76] add model revision in training_args and modify dataset loading in finetune text classification 1.add parameter model_revision in training_args.py. 2.add parameter model_revision in kwargs for finetune_text_classification.py and finetune_text_generation.py. 3.modify dataset loading in finetune_text_classification.py for flex training. Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12869552 * add model revision in training_args and modify dataset loading in finetune text classification --- .../finetune_text_classification.py | 15 +++++++++++---- .../pytorch/text_classification/run_train.sh | 19 ++++++++++++++----- .../finetune_text_generation.py | 1 + modelscope/trainers/training_args.py | 5 +++++ 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/examples/pytorch/text_classification/finetune_text_classification.py b/examples/pytorch/text_classification/finetune_text_classification.py index dfcb7b4d..0a53fb7f 100644 --- a/examples/pytorch/text_classification/finetune_text_classification.py +++ b/examples/pytorch/text_classification/finetune_text_classification.py @@ -70,16 +70,23 @@ def cfg_modify_fn(cfg): if args.dataset_json_file is None: - dataset = MsDataset.load( - args.train_dataset_name, subset_name=args.train_subset_name) - train_dataset = dataset['train'] - validation_dataset = dataset['validation'] + train_dataset = MsDataset.load( + args.train_dataset_name, + subset_name=args.train_subset_name, + split=args.train_split, + namespace=args.train_dataset_namespace) + validation_dataset = MsDataset.load( + args.val_dataset_name, + subset_name=args.val_subset_name, + split=args.val_split, + namespace=args.val_dataset_namespace) else: train_dataset, validation_dataset = build_dataset_from_file( args.dataset_json_file) kwargs = dict( model=args.model, + model_revision=args.model_revision, train_dataset=train_dataset, eval_dataset=validation_dataset, seed=args.seed, diff --git a/examples/pytorch/text_classification/run_train.sh b/examples/pytorch/text_classification/run_train.sh index e91a9996..e148655c 100644 --- a/examples/pytorch/text_classification/run_train.sh +++ b/examples/pytorch/text_classification/run_train.sh @@ -2,15 +2,24 @@ PYTHONPATH=. python examples/pytorch/text_classification/finetune_text_classific --task 'text-classification' \ --model 'damo/nlp_structbert_backbone_base_std' \ --train_dataset_name 'clue' \ + --val_dataset_name 'clue' \ --train_subset_name 'tnews' \ + --val_subset_name 'tnews' \ + --train_split 'train' \ + --val_split 'validation' \ --first_sequence 'sentence' \ - --preprocessor.label label \ - --model.num_labels 15 \ + --label label \ + --num_labels 15 \ --labels '0,1,2,3,4,5,6,7,8,9,10,11,12,13,14' \ --preprocessor 'sen-cls-tokenizer' \ --use_model_config True \ --max_epochs 1 \ - --train.dataloader.workers_per_gpu 0 \ - --evaluation.dataloader.workers_per_gpu 0 \ - --train.optimizer.lr 1e-5 \ + --per_device_train_batch_size 16 \ + --per_device_eval_batch_size 16 \ + --eval_interval 100 \ + --eval_strategy by_step \ + --work_dir './tmp' \ + --train_data_worker 0 \ + --eval_data_worker 0 \ + --lr 1e-5 \ --eval_metrics 'seq-cls-metric' \ diff --git a/examples/pytorch/text_generation/finetune_text_generation.py b/examples/pytorch/text_generation/finetune_text_generation.py index 1631911e..588d83f3 100644 --- a/examples/pytorch/text_generation/finetune_text_generation.py +++ b/examples/pytorch/text_generation/finetune_text_generation.py @@ -112,6 +112,7 @@ else: kwargs = dict( model=args.model, + model_revision=args.model_revision, train_dataset=train_dataset, eval_dataset=validation_dataset, seed=args.seed, diff --git a/modelscope/trainers/training_args.py b/modelscope/trainers/training_args.py index 77b7743e..aca3a886 100644 --- a/modelscope/trainers/training_args.py +++ b/modelscope/trainers/training_args.py @@ -117,6 +117,11 @@ class ModelArgs: 'help': 'A model id or model dir', }) + model_revision: str = field( + default=None, metadata={ + 'help': 'the revision of model', + }) + model_type: str = field( default=None, metadata={ From ae672b73df910e0afdefa6ed444fce514c147819 Mon Sep 17 00:00:00 2001 From: "hejunjie.hjj" Date: Thu, 8 Jun 2023 19:36:18 +0800 Subject: [PATCH 64/76] =?UTF-8?q?bug=20fix=20for=20single=20human=20parsin?= =?UTF-8?q?g=20Link:=20https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/code?= =?UTF-8?q?review/12869114=20*=20fix=20bug:=20single=20human=20parsing,=20?= =?UTF-8?q?=E5=BD=93src=5Fh,=20src=5Fw=E7=9B=B8=E7=AD=89=E6=97=B6=EF=BC=8C?= =?UTF-8?q?new=5Fh,=20new=5Fw=20=3D=200,=200,=20=E5=87=BA=E7=8E=B0bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modelscope/models/cv/image_human_parsing/parsing_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelscope/models/cv/image_human_parsing/parsing_utils.py b/modelscope/models/cv/image_human_parsing/parsing_utils.py index a1c20072..4d1652b4 100644 --- a/modelscope/models/cv/image_human_parsing/parsing_utils.py +++ b/modelscope/models/cv/image_human_parsing/parsing_utils.py @@ -18,7 +18,7 @@ def center_to_target_size_test(img, target_size): new_h, new_w = 0, 0 tfm_list = [] if src_h > trg_h and src_w > trg_w: - if src_h > src_w: + if src_h >= src_w: new_h = trg_h new_w = int(new_h * src_w / src_h) if new_w > trg_w: From a77d390238e4d9699a94e71a99601f61949d24bc Mon Sep 17 00:00:00 2001 From: "wucong.lyb" Date: Thu, 8 Jun 2023 19:39:05 +0800 Subject: [PATCH 65/76] fix extra_args Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12839071 * fix extra_args * update extra_args --- .../pipelines/audio/asr_inference_pipeline.py | 5 +++-- modelscope/pipelines/audio/lm_infer_pipeline.py | 5 +++-- .../audio/punctuation_processing_pipeline.py | 5 +++-- .../pipelines/audio/speaker_diarization_pipeline.py | 13 +++++++------ .../audio/speaker_verification_pipeline.py | 13 +++++++------ modelscope/pipelines/audio/timestamp_pipeline.py | 5 +++-- .../audio/voice_activity_detection_pipeline.py | 5 +++-- 7 files changed, 29 insertions(+), 22 deletions(-) diff --git a/modelscope/pipelines/audio/asr_inference_pipeline.py b/modelscope/pipelines/audio/asr_inference_pipeline.py index b9c0bd03..08b21c30 100644 --- a/modelscope/pipelines/audio/asr_inference_pipeline.py +++ b/modelscope/pipelines/audio/asr_inference_pipeline.py @@ -387,8 +387,9 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): ] for user_args in user_args_dict: - if user_args in extra_args and extra_args[user_args] is not None: - cmd[user_args] = extra_args[user_args] + if user_args in extra_args: + if extra_args.get(user_args) is not None: + cmd[user_args] = extra_args[user_args] del extra_args[user_args] return cmd diff --git a/modelscope/pipelines/audio/lm_infer_pipeline.py b/modelscope/pipelines/audio/lm_infer_pipeline.py index 75d835d6..0e252399 100644 --- a/modelscope/pipelines/audio/lm_infer_pipeline.py +++ b/modelscope/pipelines/audio/lm_infer_pipeline.py @@ -192,8 +192,9 @@ class LanguageModelPipeline(Pipeline): ] for user_args in user_args_dict: - if user_args in extra_args and extra_args[user_args] is not None: - cmd[user_args] = extra_args[user_args] + if user_args in extra_args: + if extra_args.get(user_args) is not None: + cmd[user_args] = extra_args[user_args] del extra_args[user_args] return cmd diff --git a/modelscope/pipelines/audio/punctuation_processing_pipeline.py b/modelscope/pipelines/audio/punctuation_processing_pipeline.py index 3ab3481d..53bc82ad 100644 --- a/modelscope/pipelines/audio/punctuation_processing_pipeline.py +++ b/modelscope/pipelines/audio/punctuation_processing_pipeline.py @@ -144,8 +144,9 @@ class PunctuationProcessingPipeline(Pipeline): ] for user_args in user_args_dict: - if user_args in extra_args and extra_args[user_args] is not None: - cmd[user_args] = extra_args[user_args] + if user_args in extra_args: + if extra_args.get(user_args) is not None: + cmd[user_args] = extra_args[user_args] del extra_args[user_args] return cmd diff --git a/modelscope/pipelines/audio/speaker_diarization_pipeline.py b/modelscope/pipelines/audio/speaker_diarization_pipeline.py index 71715ecd..3cf8bce7 100644 --- a/modelscope/pipelines/audio/speaker_diarization_pipeline.py +++ b/modelscope/pipelines/audio/speaker_diarization_pipeline.py @@ -199,12 +199,13 @@ class SpeakerDiarizationPipeline(Pipeline): # rewrite the config with user args for user_args in user_args_dict: - if user_args in extra_args and extra_args[user_args] is not None: - if isinstance(cmd[user_args], dict) and isinstance( - extra_args[user_args], dict): - cmd[user_args].update(extra_args[user_args]) - else: - cmd[user_args] = extra_args[user_args] + if user_args in extra_args: + if extra_args.get(user_args) is not None: + if isinstance(cmd[user_args], dict) and isinstance( + extra_args[user_args], dict): + cmd[user_args].update(extra_args[user_args]) + else: + cmd[user_args] = extra_args[user_args] del extra_args[user_args] return cmd diff --git a/modelscope/pipelines/audio/speaker_verification_pipeline.py b/modelscope/pipelines/audio/speaker_verification_pipeline.py index e576885a..7eace5ea 100644 --- a/modelscope/pipelines/audio/speaker_verification_pipeline.py +++ b/modelscope/pipelines/audio/speaker_verification_pipeline.py @@ -166,12 +166,13 @@ class SpeakerVerificationPipeline(Pipeline): # rewrite the config with user args for user_args in user_args_dict: - if user_args in extra_args and extra_args[user_args] is not None: - if isinstance(cmd[user_args], dict) and isinstance( - extra_args[user_args], dict): - cmd[user_args].update(extra_args[user_args]) - else: - cmd[user_args] = extra_args[user_args] + if user_args in extra_args: + if extra_args.get(user_args) is not None: + if isinstance(cmd[user_args], dict) and isinstance( + extra_args[user_args], dict): + cmd[user_args].update(extra_args[user_args]) + else: + cmd[user_args] = extra_args[user_args] del extra_args[user_args] return cmd diff --git a/modelscope/pipelines/audio/timestamp_pipeline.py b/modelscope/pipelines/audio/timestamp_pipeline.py index 0968b359..d21207d9 100644 --- a/modelscope/pipelines/audio/timestamp_pipeline.py +++ b/modelscope/pipelines/audio/timestamp_pipeline.py @@ -267,8 +267,9 @@ class TimestampPipeline(Pipeline): ] for user_args in user_args_dict: - if user_args in extra_args and extra_args[user_args] is not None: - cmd[user_args] = extra_args[user_args] + if user_args in extra_args: + if extra_args.get(user_args) is not None: + cmd[user_args] = extra_args[user_args] del extra_args[user_args] return cmd diff --git a/modelscope/pipelines/audio/voice_activity_detection_pipeline.py b/modelscope/pipelines/audio/voice_activity_detection_pipeline.py index 0121b242..2ffa2181 100644 --- a/modelscope/pipelines/audio/voice_activity_detection_pipeline.py +++ b/modelscope/pipelines/audio/voice_activity_detection_pipeline.py @@ -212,8 +212,9 @@ class VoiceActivityDetectionPipeline(Pipeline): ] for user_args in user_args_dict: - if user_args in extra_args and extra_args[user_args] is not None: - cmd[user_args] = extra_args[user_args] + if user_args in extra_args: + if extra_args.get(user_args) is not None: + cmd[user_args] = extra_args[user_args] del extra_args[user_args] return cmd From 96c2d42f09c82598dcb2a38cb18c9a77a46ec90c Mon Sep 17 00:00:00 2001 From: "hemu.zp" Date: Thu, 8 Jun 2023 19:40:14 +0800 Subject: [PATCH 66/76] Add StreamingMixin Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12445731 * StreamingMixin poc * update design * Merge branch 'master' into feat/StreamingMixin * add dicstr * make postprocessor input consistent --- modelscope/models/nlp/gpt3/backbone.py | 12 +- .../models/nlp/gpt3/distributed_gpt3.py | 19 ++- modelscope/models/nlp/gpt3/text_generation.py | 11 +- .../nlp/distributed_gpt3_pipeline.py | 44 +++++- .../pipelines/nlp/text_generation_pipeline.py | 3 +- modelscope/utils/streaming_output.py | 145 ++++++++++++++++++ tests/pipelines/test_gpt3_text_generation.py | 7 + tests/pipelines/test_text_generation.py | 32 ++++ tests/utils/test_ast.py | 2 +- 9 files changed, 260 insertions(+), 15 deletions(-) create mode 100644 modelscope/utils/streaming_output.py diff --git a/modelscope/models/nlp/gpt3/backbone.py b/modelscope/models/nlp/gpt3/backbone.py index 2f8e4699..89e19480 100644 --- a/modelscope/models/nlp/gpt3/backbone.py +++ b/modelscope/models/nlp/gpt3/backbone.py @@ -353,7 +353,7 @@ class GPT3Model(PreTrainedModel): model.load_state_dict(state_dict) return model - def generate(self, tokens, temperature=1.0, **kwargs): + def streaming_generate(self, tokens, temperature=1.0, **kwargs): top_k = kwargs.pop('top_k', self.config.top_k) top_p = kwargs.pop('top_p', self.config.top_p) max_length = kwargs.pop('max_length', tokens.size(1) + 100) @@ -410,6 +410,9 @@ class GPT3Model(PreTrainedModel): # Update the tokens. tokens[started, context_length] = new_sample[started] + yield TokenGeneratorOutput(sequences=tokens[:, :(context_length + + 1)]) + done_token = (new_sample == termination_id).byte() & \ started.byte() @@ -419,5 +422,8 @@ class GPT3Model(PreTrainedModel): if done: break - tokens = tokens[:, :(context_length + 1)] - return TokenGeneratorOutput(sequences=tokens) + def generate(self, tokens, temperature=1.0, **kwargs): + last_output = None + for output in self.streaming_generate(tokens, temperature, **kwargs): + last_output = output + return last_output diff --git a/modelscope/models/nlp/gpt3/distributed_gpt3.py b/modelscope/models/nlp/gpt3/distributed_gpt3.py index 538631b6..570487df 100644 --- a/modelscope/models/nlp/gpt3/distributed_gpt3.py +++ b/modelscope/models/nlp/gpt3/distributed_gpt3.py @@ -33,6 +33,7 @@ from modelscope.models.nlp.gpt3 import GPT3Config from modelscope.outputs import TextGenerationModelOutput, TokenGeneratorOutput from modelscope.utils.megatron_utils import init_megatron_util from modelscope.utils.nlp.load_checkpoint import pre_load +from modelscope.utils.streaming_output import StreamingOutputMixin class GPT3ParallelMLP(nn.Module): @@ -945,7 +946,7 @@ def split_state_dict(state_dict: Dict[str, torch.Tensor], model: GPT3Model, return state_dict -class DistributedGPT3(TorchModel): +class DistributedGPT3(TorchModel, StreamingOutputMixin): def __init__(self, model_dir, @@ -1104,6 +1105,10 @@ class DistributedGPT3(TorchModel): # Update the tokens. tokens[started, context_length] = new_sample[started] + # streaming output + yield TokenGeneratorOutput(sequences=tokens[:, :(context_length + + 1)]) + # Update the context length for the next token generation. prev_context_length = context_length @@ -1128,9 +1133,6 @@ class DistributedGPT3(TorchModel): if use_eod_token_for_early_termination and done: break - tokens = tokens[:, :(context_length + 1)] - return TokenGeneratorOutput(sequences=tokens) - def beam_search(self, tokens, beam_size=5, num_return_gen=1, **kwargs): batch_size = tokens.size(0) assert (batch_size == 1) @@ -1247,10 +1249,17 @@ class DistributedGPT3(TorchModel): @torch.no_grad() def generate(self, tokens, do_sample=True, *args, **kwargs): if do_sample: - return self.sample(tokens, *args, **kwargs) + last_output = None + for output in self.sample(tokens, *args, **kwargs): + last_output = output + return last_output else: return self.beam_search(tokens, *args, **kwargs) + @torch.no_grad() + def stream(self, tokens, *args, **kwargs): + return self.sample(tokens, *args, **kwargs) + def state_dict(self, destination=None, prefix='', keep_vars=False): return self.dist_model.state_dict(destination, prefix, keep_vars) diff --git a/modelscope/models/nlp/gpt3/text_generation.py b/modelscope/models/nlp/gpt3/text_generation.py index fbc82b8a..956e4553 100644 --- a/modelscope/models/nlp/gpt3/text_generation.py +++ b/modelscope/models/nlp/gpt3/text_generation.py @@ -1,6 +1,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from collections import OrderedDict -from typing import Dict +from typing import Dict, Generator import torch from transformers import BertTokenizer @@ -11,12 +11,13 @@ from modelscope.models.builder import MODELS from modelscope.models.nlp.gpt3 import GPT3Model from modelscope.utils.constant import Tasks from modelscope.utils.hub import read_config +from modelscope.utils.streaming_output import StreamingOutputMixin __all__ = ['GPT3ForTextGeneration'] @MODELS.register_module(Tasks.text_generation, module_name=Models.gpt3) -class GPT3ForTextGeneration(TorchModel): +class GPT3ForTextGeneration(TorchModel, StreamingOutputMixin): def __init__(self, model_dir: str, *args, **kwargs): """initialize the text generation model from the `model_dir` path. @@ -77,3 +78,9 @@ class GPT3ForTextGeneration(TorchModel): state_dict: 'OrderedDict[str, Tensor]', strict: bool = True): return self.model.load_state_dict(state_dict, strict) + + def stream(self, inputs, **kwargs) -> Generator: + tokens = inputs['input_ids'] + lengths = self._get_length(inputs['attention_mask']) + return self.model.streaming_generate( + tokens, prompt_length=lengths, **kwargs) diff --git a/modelscope/pipelines/nlp/distributed_gpt3_pipeline.py b/modelscope/pipelines/nlp/distributed_gpt3_pipeline.py index 1738f2da..6962a09a 100644 --- a/modelscope/pipelines/nlp/distributed_gpt3_pipeline.py +++ b/modelscope/pipelines/nlp/distributed_gpt3_pipeline.py @@ -1,6 +1,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -from typing import Any, Dict +from typing import Any, Dict, Generator, Optional import torch @@ -9,12 +9,15 @@ from modelscope.models.nlp import DistributedGPT3 from modelscope.pipelines.base import DistributedPipeline from modelscope.pipelines.builder import PIPELINES from modelscope.preprocessors import TextGenerationJiebaPreprocessor -from modelscope.utils.constant import Tasks +from modelscope.utils.constant import Frameworks, Tasks +from modelscope.utils.device import device_placement +from modelscope.utils.streaming_output import PipelineStreamingOutputMixin @PIPELINES.register_module( Tasks.text_generation, module_name=Pipelines.gpt3_generation) -class DistributedGPT3Pipeline(DistributedPipeline): +class DistributedGPT3Pipeline(DistributedPipeline, + PipelineStreamingOutputMixin): """This class is used to instantiate the gpt3 model. """ @@ -33,6 +36,8 @@ class DistributedGPT3Pipeline(DistributedPipeline): preprocessor = TextGenerationJiebaPreprocessor(model) super().__init__(model, preprocessor=preprocessor, **kwargs) assert hasattr(preprocessor, 'tokenizer') + self.model = PipelineStreamingOutputMixin() + self._model_prepare = True @classmethod def _instantiate_one(cls, rank, model_dir, **kwargs): @@ -64,3 +69,36 @@ class DistributedGPT3Pipeline(DistributedPipeline): def _sanitize_parameters(self, **pipeline_parameters): return {}, pipeline_parameters, {} + + def _stream_single(self, model_input: Dict[str, Any], + forward_params: Dict[str, Any], + postprocess_params: Dict[str, Any]) -> Generator: + + with device_placement(self.framework, self.device_name): + if self._auto_collate: + model_input = self._collate_fn(model_input) + inputs = {'inputs': model_input, 'forward_params': forward_params} + self.model_pool.map(self.__class__._stream_one, + [inputs] * self.world_size) + + while True: + res = self.model_pool.map(self.__class__._next_one, + range(self.world_size)) + if res[0] is None: + break + out = self.postprocess(res[0], **postprocess_params) + self._check_output(out) + yield out + + @classmethod + def _stream_one(cls, inputs: Dict[str, Any]) -> None: + tokens = inputs['inputs']['input_ids'].cuda( + torch.cuda.current_device()) + cls._stream = cls.model.stream(tokens, **inputs['forward_params']) + + @classmethod + def _next_one(cls, idx: int) -> Optional[Dict[str, Any]]: + try: + return next(cls._stream) + except StopIteration: + return None diff --git a/modelscope/pipelines/nlp/text_generation_pipeline.py b/modelscope/pipelines/nlp/text_generation_pipeline.py index d1aa5ff6..cfc3645d 100644 --- a/modelscope/pipelines/nlp/text_generation_pipeline.py +++ b/modelscope/pipelines/nlp/text_generation_pipeline.py @@ -15,13 +15,14 @@ from modelscope.preprocessors import Preprocessor from modelscope.utils.chinese_utils import remove_space_between_chinese_chars from modelscope.utils.constant import ModelFile, Tasks from modelscope.utils.hub import Config, read_config +from modelscope.utils.streaming_output import PipelineStreamingOutputMixin __all__ = ['TextGenerationPipeline', 'TextGenerationT5Pipeline'] @PIPELINES.register_module( Tasks.text_generation, module_name=Pipelines.text_generation) -class TextGenerationPipeline(Pipeline): +class TextGenerationPipeline(Pipeline, PipelineStreamingOutputMixin): def __init__(self, model: Union[Model, str], diff --git a/modelscope/utils/streaming_output.py b/modelscope/utils/streaming_output.py new file mode 100644 index 00000000..1a011a62 --- /dev/null +++ b/modelscope/utils/streaming_output.py @@ -0,0 +1,145 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Any, Dict, Generator, List, Union + +import torch + +from modelscope.pipelines.base import Input +from modelscope.utils.constant import Frameworks +from modelscope.utils.device import device_placement + + +class StreamingOutputMixin: + + def stream(self, *args, **kwargs) -> Generator: + """ + Support the input of Model and Pipeline. + The output is a `Generator` type, + which conforms to the output standard of modelscope. + """ + raise NotImplementedError + + +class PipelineStreamingOutputMixin(StreamingOutputMixin): + + def stream(self, input: Union[Input, List[Input]], *args, + **kwargs) -> Generator: + """ + Similar to the `Pipeline.__call__` method. + it supports the input that the pipeline can accept, + and also supports batch input. + + self.model must be a subclass of StreamingOutputMixin + and implement the stream method. + """ + assert isinstance(self.model, StreamingOutputMixin + ), 'pipeline.model must be StreamingOutputMixin!' + if (self.model or (self.has_multiple_models and self.models[0])): + if not self._model_prepare: + self.prepare_model() + + batch_size = kwargs.pop('batch_size', None) + preprocess_params, forward_params, postprocess_params = self._sanitize_parameters( + **kwargs) + + if isinstance(input, list): + model_input_list = [ + self._preprocess_with_check(i, preprocess_params) + for i in input + ] + + if batch_size is None: + output = [] + for ele in model_input_list: + output.append( + self._stream_single(ele, forward_params, + postprocess_params)) + else: + output = self._stream_batch(model_input_list, batch_size, + forward_params, postprocess_params) + + else: + model_input = self._preprocess_with_check(input, preprocess_params) + output = self._stream_single(model_input, forward_params, + postprocess_params) + return output + + def _preprocess_with_check( + self, input: Input, + preprocess_params: Dict[str, Any]) -> Dict[str, Any]: + self._check_input(input) + return self.preprocess(input, **preprocess_params) + + def _stream_single(self, model_input: Dict[str, Any], + forward_params: Dict[str, Any], + postprocess_params: Dict[str, Any]) -> Generator: + + with device_placement(self.framework, self.device_name): + if self.framework == Frameworks.torch: + with torch.no_grad(): + if self._auto_collate: + model_input = self._collate_fn(model_input) + stream = self.model.stream(model_input, **forward_params) + else: + stream = self.model.stream(model_input, **forward_params) + + for out in stream: + out = self.postprocess(out, **postprocess_params) + self._check_output(out) + yield out + + def _stream_batch(self, model_input_list: List[Dict[str, Any]], + batch_size: int, forward_params: Dict[str, Any], + postprocess_params: Dict[str, Any]) -> Generator: + + stream_list = [] + real_batch_sizes = [] + with device_placement(self.framework, self.device_name): + for i in range(0, len(model_input_list), batch_size): + end = min(i + batch_size, len(model_input_list)) + real_batch_size = end - i + real_batch_sizes.append(real_batch_size) + + batched_out = self._batch(model_input_list[i:end]) + if self.framework == Frameworks.torch: + with torch.no_grad(): + if self._auto_collate: + batched_out = self._collate_fn(batched_out) + stream_list.append( + self.model.stream(batched_out, **forward_params)) + else: + stream_list.append( + self.model.stream(batched_out, **forward_params)) + + output_list = [None] * len(model_input_list) + stop_streams = 0 + while stop_streams < len(stream_list): + stop_streams = 0 + for i, (stream, real_batch_size) in enumerate( + zip(stream_list, real_batch_sizes)): + try: + batched_out = next(stream) + for batch_idx in range(real_batch_size): + out = {} + for k, element in batched_out.items(): + if element is not None: + if isinstance(element, (tuple, list)): + if isinstance(element[0], + torch.Tensor): + out[k] = type(element)( + e[batch_idx:batch_idx + 1] + for e in element) + else: + # Compatible with traditional pipelines + out[k] = element[batch_idx] + else: + out[k] = element[batch_idx:batch_idx + + 1] + out = self.postprocess(out, **postprocess_params) + self._check_output(out) + output_index = i * batch_size + batch_idx + output_list[output_index] = out + except StopIteration: + stop_streams += 1 + yield output_list + + return output_list diff --git a/tests/pipelines/test_gpt3_text_generation.py b/tests/pipelines/test_gpt3_text_generation.py index 1d938384..832295de 100644 --- a/tests/pipelines/test_gpt3_text_generation.py +++ b/tests/pipelines/test_gpt3_text_generation.py @@ -22,6 +22,13 @@ class TextGPT3GenerationTest(unittest.TestCase): pipe = pipeline(Tasks.text_generation, model=self.model_id_1_3B) print(pipe(self.input)) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_gpt3_1_3B_with_streaming(self): + pipe = pipeline(Tasks.text_generation, model=self.model_id_1_3B) + for output in pipe.stream(self.input, max_length=64): + print(output, end='\r') + print() + @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') def test_gpt3_2_7B(self): pipe = pipeline(Tasks.text_generation, model=self.model_id_2_7B) diff --git a/tests/pipelines/test_text_generation.py b/tests/pipelines/test_text_generation.py index 378b1bbc..687a38df 100644 --- a/tests/pipelines/test_text_generation.py +++ b/tests/pipelines/test_text_generation.py @@ -8,6 +8,7 @@ from modelscope.pipelines import pipeline from modelscope.pipelines.nlp import TextGenerationPipeline from modelscope.preprocessors import TextGenerationTransformersPreprocessor from modelscope.utils.constant import Tasks +from modelscope.utils.streaming_output import StreamingOutputMixin from modelscope.utils.test_utils import test_level @@ -61,6 +62,20 @@ class TextGenerationTest(unittest.TestCase): task=Tasks.text_generation, model=model_id, **init_kwargs) print(pipeline_ins(input, **run_kwargs)) + def run_streaming_pipeline_with_model_id(self, + model_id, + input, + init_kwargs={}, + run_kwargs={}): + pipeline_ins = pipeline( + task=Tasks.text_generation, model=model_id, **init_kwargs) + + # set stream inputs + assert isinstance(pipeline_ins, StreamingOutputMixin) + for output in pipeline_ins.stream(input, **run_kwargs): + print(output, end='\r') + print() + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_palm_zh_base_with_model_name(self): self.run_pipeline_with_model_id(self.palm_model_id_zh_base, @@ -123,6 +138,23 @@ class TextGenerationTest(unittest.TestCase): [self.gpt3_input, self.gpt3_input[:10], self.gpt3_input[10:]], run_kwargs={'batch_size': 2}) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_gpt_base_with_model_name_with_streaming(self): + self.run_streaming_pipeline_with_model_id( + self.gpt3_base_model_id, + self.gpt3_input, + run_kwargs={'max_length': 64}) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_gpt_base_with_model_name_with_streaming_batch(self): + self.run_streaming_pipeline_with_model_id( + self.gpt3_base_model_id, + [self.gpt3_input, self.gpt3_input[:10], self.gpt3_input[10:]], + run_kwargs={ + 'batch_size': 2, + 'max_length': 32 + }) + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_gpt_base_with_model_name_batch_iter(self): self.run_pipeline_with_model_id( diff --git a/tests/utils/test_ast.py b/tests/utils/test_ast.py index 78917ccd..288c076a 100644 --- a/tests/utils/test_ast.py +++ b/tests/utils/test_ast.py @@ -47,7 +47,7 @@ class AstScaningTest(unittest.TestCase): self.assertIsInstance(decorators, list) self.assertListEqual( list(set(imports.keys()) - set(['torch', 'os'])), []) - self.assertEqual(len(from_imports.keys()), 10) + self.assertEqual(len(from_imports.keys()), 11) self.assertTrue(from_imports['modelscope.metainfo'] is not None) self.assertEqual(from_imports['modelscope.metainfo'], ['Pipelines']) self.assertEqual( From d6bf9e80fb278f2b8c38506a56f30f1522d207c0 Mon Sep 17 00:00:00 2001 From: "mulin.lyh" Date: Mon, 12 Jun 2023 11:03:28 +0800 Subject: [PATCH 67/76] [to #50129142]feat: support pre build torch extension build image, first extension megatron_util --- .dev_scripts/build_image.sh | 4 ++- .../pipelines/cv/card_detection_pipeline.py | 2 +- .../pipelines/cv/face_detection_pipeline.py | 2 +- .../{typing.py => input_output_typing.py} | 0 modelscope/utils/pre_compile.py | 27 +++++++++++++++++++ 5 files changed, 32 insertions(+), 3 deletions(-) rename modelscope/utils/{typing.py => input_output_typing.py} (100%) create mode 100644 modelscope/utils/pre_compile.py diff --git a/.dev_scripts/build_image.sh b/.dev_scripts/build_image.sh index 948ecb64..a9cbef58 100644 --- a/.dev_scripts/build_image.sh +++ b/.dev_scripts/build_image.sh @@ -132,7 +132,7 @@ echo -e "Building image with:\npython$python_version\npytorch$torch_version\nten docker_file_content=`cat docker/Dockerfile.ubuntu` if [ "$is_ci_test" != "True" ]; then echo "Building ModelScope lib, will install ModelScope lib to image" - docker_file_content="${docker_file_content} \nRUN pip install --no-cache-dir modelscope==$modelscope_version -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html" + docker_file_content="${docker_file_content} \nRUN pip install --no-cache-dir modelscope==$modelscope_version -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html" fi echo "$is_dsw" if [ "$is_dsw" == "False" ]; then @@ -140,6 +140,8 @@ if [ "$is_dsw" == "False" ]; then else echo "Building dsw image will need set ModelScope lib cache location." docker_file_content="${docker_file_content} \nENV MODELSCOPE_CACHE=/mnt/workspace/.cache/modelscope" + # pre compile extension + docker_file_content="${docker_file_content} \nRUN python -c 'from modelscope.utils.pre_compile import pre_compile_all;pre_compile_all()'" fi if [ "$is_ci_test" == "True" ]; then echo "Building CI image, uninstall modelscope" diff --git a/modelscope/pipelines/cv/card_detection_pipeline.py b/modelscope/pipelines/cv/card_detection_pipeline.py index 75e7743a..5d7e87b0 100644 --- a/modelscope/pipelines/cv/card_detection_pipeline.py +++ b/modelscope/pipelines/cv/card_detection_pipeline.py @@ -6,8 +6,8 @@ from modelscope.models.base.base_model import Model from modelscope.pipelines.base import Pipeline from modelscope.pipelines.builder import PIPELINES from modelscope.utils.constant import Tasks +from modelscope.utils.input_output_typing import Image from modelscope.utils.logger import get_logger -from modelscope.utils.typing import Image logger = get_logger() diff --git a/modelscope/pipelines/cv/face_detection_pipeline.py b/modelscope/pipelines/cv/face_detection_pipeline.py index 58aefedf..115e8998 100644 --- a/modelscope/pipelines/cv/face_detection_pipeline.py +++ b/modelscope/pipelines/cv/face_detection_pipeline.py @@ -16,8 +16,8 @@ from modelscope.pipelines.builder import PIPELINES from modelscope.preprocessors import LoadImage from modelscope.utils.config import Config from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.input_output_typing import Image from modelscope.utils.logger import get_logger -from modelscope.utils.typing import Image logger = get_logger() diff --git a/modelscope/utils/typing.py b/modelscope/utils/input_output_typing.py similarity index 100% rename from modelscope/utils/typing.py rename to modelscope/utils/input_output_typing.py diff --git a/modelscope/utils/pre_compile.py b/modelscope/utils/pre_compile.py new file mode 100644 index 00000000..65bde30a --- /dev/null +++ b/modelscope/utils/pre_compile.py @@ -0,0 +1,27 @@ +import os + +import torch + +from modelscope.utils.megatron_utils import init_megatron_util + + +def pre_compile_megatron_util(): + dummy_megatron_cfg = { + 'tensor_model_parallel_size': 1, + 'world_size': 1, + 'distributed_backend': 'nccl', + 'seed': 42, + } + os.environ['MASTER_PORT'] = '39501' + init_megatron_util(dummy_megatron_cfg) + + +def pre_compile_all(): + if torch.cuda.is_available(): # extension require cuda. + pre_compile_megatron_util() + + # extension for all platform. + + +if __name__ == '__main__': + pre_compile_all() From 8b4e9dcdfbe71e973e21557da062259b5a77aed1 Mon Sep 17 00:00:00 2001 From: "zhifu.gzf" Date: Tue, 13 Jun 2023 16:36:21 +0800 Subject: [PATCH 68/76] ngpu bug and rm easyasr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复ngpu指定无效的问题；移除easyasr，全部涉及模型都下架了；将funasr版本限制为>=0.6.0 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12933049 * ngpu bug and rm easyasr --- modelscope/pipelines/audio/asr_inference_pipeline.py | 2 +- modelscope/pipelines/audio/lm_infer_pipeline.py | 2 +- modelscope/pipelines/audio/punctuation_processing_pipeline.py | 2 +- modelscope/pipelines/audio/speaker_diarization_pipeline.py | 2 +- modelscope/pipelines/audio/speaker_verification_pipeline.py | 2 +- modelscope/pipelines/audio/timestamp_pipeline.py | 2 +- .../pipelines/audio/voice_activity_detection_pipeline.py | 2 +- requirements/audio/audio_asr.txt | 3 +-- 8 files changed, 8 insertions(+), 9 deletions(-) diff --git a/modelscope/pipelines/audio/asr_inference_pipeline.py b/modelscope/pipelines/audio/asr_inference_pipeline.py index 08b21c30..823964e5 100644 --- a/modelscope/pipelines/audio/asr_inference_pipeline.py +++ b/modelscope/pipelines/audio/asr_inference_pipeline.py @@ -127,7 +127,7 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): minlenratio=self.cmd['minlenratio'], batch_size=self.cmd['batch_size'], beam_size=self.cmd['beam_size'], - ngpu=self.cmd['ngpu'], + ngpu=ngpu, ctc_weight=self.cmd['ctc_weight'], lm_weight=self.cmd['lm_weight'], penalty=self.cmd['penalty'], diff --git a/modelscope/pipelines/audio/lm_infer_pipeline.py b/modelscope/pipelines/audio/lm_infer_pipeline.py index 0e252399..e1524ebd 100644 --- a/modelscope/pipelines/audio/lm_infer_pipeline.py +++ b/modelscope/pipelines/audio/lm_infer_pipeline.py @@ -80,7 +80,7 @@ class LanguageModelPipeline(Pipeline): mode=self.cmd['mode'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=self.cmd['ngpu'], + ngpu=ngpu, seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], diff --git a/modelscope/pipelines/audio/punctuation_processing_pipeline.py b/modelscope/pipelines/audio/punctuation_processing_pipeline.py index 53bc82ad..4e41e0c0 100644 --- a/modelscope/pipelines/audio/punctuation_processing_pipeline.py +++ b/modelscope/pipelines/audio/punctuation_processing_pipeline.py @@ -54,7 +54,7 @@ class PunctuationProcessingPipeline(Pipeline): mode=self.cmd['mode'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=self.cmd['ngpu'], + ngpu=ngpu, seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], diff --git a/modelscope/pipelines/audio/speaker_diarization_pipeline.py b/modelscope/pipelines/audio/speaker_diarization_pipeline.py index 3cf8bce7..a20cfcad 100644 --- a/modelscope/pipelines/audio/speaker_diarization_pipeline.py +++ b/modelscope/pipelines/audio/speaker_diarization_pipeline.py @@ -77,7 +77,7 @@ class SpeakerDiarizationPipeline(Pipeline): output_dir=self.cmd['output_dir'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=self.cmd['ngpu'], + ngpu=ngpu, seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], diff --git a/modelscope/pipelines/audio/speaker_verification_pipeline.py b/modelscope/pipelines/audio/speaker_verification_pipeline.py index 7eace5ea..c10f6a95 100644 --- a/modelscope/pipelines/audio/speaker_verification_pipeline.py +++ b/modelscope/pipelines/audio/speaker_verification_pipeline.py @@ -57,7 +57,7 @@ class SpeakerVerificationPipeline(Pipeline): output_dir=self.cmd['output_dir'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=self.cmd['ngpu'], + ngpu=ngpu, seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], diff --git a/modelscope/pipelines/audio/timestamp_pipeline.py b/modelscope/pipelines/audio/timestamp_pipeline.py index d21207d9..17cf9545 100644 --- a/modelscope/pipelines/audio/timestamp_pipeline.py +++ b/modelscope/pipelines/audio/timestamp_pipeline.py @@ -75,7 +75,7 @@ class TimestampPipeline(Pipeline): mode=self.cmd['mode'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=self.cmd['ngpu'], + ngpu=ngpu, seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], diff --git a/modelscope/pipelines/audio/voice_activity_detection_pipeline.py b/modelscope/pipelines/audio/voice_activity_detection_pipeline.py index 2ffa2181..3e00454a 100644 --- a/modelscope/pipelines/audio/voice_activity_detection_pipeline.py +++ b/modelscope/pipelines/audio/voice_activity_detection_pipeline.py @@ -56,7 +56,7 @@ class VoiceActivityDetectionPipeline(Pipeline): mode=self.cmd['mode'], batch_size=self.cmd['batch_size'], dtype=self.cmd['dtype'], - ngpu=self.cmd['ngpu'], + ngpu=ngpu, seed=self.cmd['seed'], num_workers=self.cmd['num_workers'], log_level=self.cmd['log_level'], diff --git a/requirements/audio/audio_asr.txt b/requirements/audio/audio_asr.txt index 7725a0dd..d136ce55 100644 --- a/requirements/audio/audio_asr.txt +++ b/requirements/audio/audio_asr.txt @@ -1,2 +1 @@ -easyasr>=0.0.2 -funasr>=0.5.0 +funasr>=0.6.0 From de491cfe6b770cc84c5f1a486755faacefeff241 Mon Sep 17 00:00:00 2001 From: "suluyan.sly" Date: Wed, 14 Jun 2023 10:00:23 +0800 Subject: [PATCH 69/76] [to #48855014] llama finetune + deepspeed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. llama base finetune：使用trainer从llama finetune至alpaca，效果确认并提供example 2. deepspeed通用性完善：mpu解耦；deepspeed的训练信息支持从ms log透出（目前ms打印的log有误）；支持从modelscope configuration.json 进行 deepspeed config 配置；deepspeed optimizer和lr_scheduler 初始化支持；解决deepspeed和ddp同时使用报错；解决保存ckpt时报错 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12651323 --- examples/pytorch/llama/finetune_llama.py | 263 ++++++++++++++++++ examples/pytorch/llama/run_train_llama.sh | 9 + .../hooks/distributed/deepspeed_hook.py | 237 ++++++++++++++-- .../trainers/hooks/lr_scheduler_hook.py | 30 +- modelscope/trainers/trainer.py | 5 +- 5 files changed, 497 insertions(+), 47 deletions(-) create mode 100644 examples/pytorch/llama/finetune_llama.py create mode 100644 examples/pytorch/llama/run_train_llama.sh diff --git a/examples/pytorch/llama/finetune_llama.py b/examples/pytorch/llama/finetune_llama.py new file mode 100644 index 00000000..88975e66 --- /dev/null +++ b/examples/pytorch/llama/finetune_llama.py @@ -0,0 +1,263 @@ +# Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li +# Copyright (c) Alibaba, Inc. and its affiliates. + +import copy +import logging +import os +import shutil +import tempfile +import unittest +from dataclasses import dataclass, field + +import json +import torch +import utils + +from modelscope import TrainingArgs +from modelscope.hub.snapshot_download import snapshot_download +from modelscope.metainfo import Trainers +from modelscope.models.nlp.llama import LlamaForTextGeneration, LlamaTokenizer +from modelscope.msdatasets.dataset_cls.custom_datasets.torch_custom_dataset import \ + TorchCustomDataset +from modelscope.trainers import build_trainer + +IGNORE_INDEX = -100 +DEFAULT_PAD_TOKEN = '[PAD]' +DEFAULT_EOS_TOKEN = '' +DEFAULT_BOS_TOKEN = '' +DEFAULT_UNK_TOKEN = '' +PROMPT_DICT = { + 'prompt_input': + ('Below is an instruction that describes a task, paired with an input that provides further context. ' + 'Write a response that appropriately completes the request.\n\n' + '### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:' + ), + 'prompt_no_input': + ('Below is an instruction that describes a task. ' + 'Write a response that appropriately completes the request.\n\n' + '### Instruction:\n{instruction}\n\n### Response:'), +} + + +@dataclass(init=False) +class TextGenerationArguments(TrainingArgs): + src_txt: str = field( + default=None, + metadata={ + 'help': 'The source text key of preprocessor', + 'cfg_node': 'preprocessor.src_txt' + }) + + deepspeed: str = field( + default=None, + metadata={ + 'help': 'The location of DeepSpeed json config file.', + }) + + work_dir: str = field( + default=None, metadata={ + 'help': 'The location of work dir', + }) + + +def _tokenize_fn(strings, tokenizer): + """Tokenize a list of strings.""" + tokenized_list = [ + tokenizer( + text, + return_tensors='pt', + padding='longest', + max_length=tokenizer.model_max_length, + truncation=True, + ) for text in strings + ] + input_ids = labels = [ + tokenized.input_ids[0] for tokenized in tokenized_list + ] + input_ids_lens = labels_lens = [ + tokenized.input_ids.ne(tokenizer.pad_token_id).sum().item() + for tokenized in tokenized_list + ] + return dict( + input_ids=input_ids, + labels=labels, + input_ids_lens=input_ids_lens, + labels_lens=labels_lens, + ) + + +def preprocess(sources, targets, tokenizer): + """Preprocess the data by tokenizing.""" + examples = [s + t for s, t in zip(sources, targets)] + examples_tokenized, sources_tokenized = [ + _tokenize_fn(strings, tokenizer) for strings in (examples, sources) + ] + input_ids = examples_tokenized['input_ids'] + labels = copy.deepcopy(input_ids) + for label, source_len in zip(labels, sources_tokenized['input_ids_lens']): + label[:source_len] = IGNORE_INDEX + return dict(input_ids=input_ids, labels=labels) + + +def smart_tokenizer_and_embedding_resize(special_tokens_dict, tokenizer, + model): + """Resize tokenizer and embedding. + + Note: This is the unoptimized version that may make your embedding size not be divisible by 64. + """ + num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict) + model.resize_token_embeddings(len(tokenizer)) + + if num_new_tokens > 0: + input_embeddings = model.get_input_embeddings().weight.data + output_embeddings = model.get_output_embeddings().weight.data + + input_embeddings_avg = input_embeddings[:-num_new_tokens].mean( + dim=0, keepdim=True) + output_embeddings_avg = output_embeddings[:-num_new_tokens].mean( + dim=0, keepdim=True) + + input_embeddings[-num_new_tokens:] = input_embeddings_avg + output_embeddings[-num_new_tokens:] = output_embeddings_avg + + +class SupervisedDataset(TorchCustomDataset): + """Dataset for supervised fine-tuning.""" + + def __init__(self, data_path: str, tokenizer): + logging.warning('Loading data...') + f = open(data_path, 'r') + list_data_dict = json.load(f) + f.close() + + logging.warning('Formatting inputs...') + prompt_input, prompt_no_input = PROMPT_DICT[ + 'prompt_input'], PROMPT_DICT['prompt_no_input'] + sources = [ + prompt_input.format_map(example) if example.get('input', '') != '' + else prompt_no_input.format_map(example) + for example in list_data_dict + ] + targets = [ + f"{example['output']}{tokenizer.eos_token}" + for example in list_data_dict + ] + + logging.warning('Tokenizing inputs... This may take some time...') + data_dict = preprocess(sources, targets, tokenizer) + + self.input_ids = data_dict['input_ids'] + self.labels = data_dict['labels'] + + def __len__(self): + return len(self.input_ids) + + def __getitem__(self, i): + return dict(input_ids=self.input_ids[i], labels=self.labels[i]) + + +@dataclass +class DataCollatorForSupervisedDataset(object): + """Collate examples for supervised fine-tuning.""" + + tokenizer: LlamaTokenizer + + def __call__(self, instances): + input_ids, labels = tuple([instance[key] for instance in instances] + for key in ('input_ids', 'labels')) + input_ids = torch.nn.utils.rnn.pad_sequence( + input_ids, + batch_first=True, + padding_value=self.tokenizer.pad_token_id) + labels = torch.nn.utils.rnn.pad_sequence( + labels, batch_first=True, padding_value=IGNORE_INDEX) + return dict( + input_ids=input_ids, + labels=labels, + attention_mask=input_ids.ne(self.tokenizer.pad_token_id), + ) + + +config, args = TextGenerationArguments().parse_cli().to_config() + +if __name__ == '__main__': + + def cfg_modify_fn(cfg): + if args.use_model_config: + cfg.merge_from_dict(config) + else: + cfg = config + cfg.train.lr_scheduler = { + 'type': 'CosineAnnealingLR', + 'T_max': 1, + 'options': { + 'by_epoch': False + } + } + cfg.train.optimizer = { + 'type': 'AdamW', + 'lr': 2e-5, + 'weight_decay': 0.0, + 'options': { + 'cumulative_iters': 8, + 'warmup': { + 'type': 'LinearWarmup', + 'warmup_ratio': 0.03 + } + } + } + cfg.train.logging = {'interval': 8, 'by_epoch': False} + cfg.train['bf16'] = True + cfg.train.dataloader = {'batch_size_per_gpu': 4, 'workers_per_gpu': 1} + if 'hooks' not in cfg.train: + cfg.train['hooks'] = [] + cfg.train.hooks.append({ + 'type': 'DeepspeedHook', + 'config': args.deepspeed, + 'save_zero_checkpoint': True, + 'with_mpu': False, + }) + + cfg.preprocessor.sequence_length = 512 + return cfg + + model_path = args.model if os.path.exists( + args.model) else snapshot_download(args.model) + data_path = args.src_txt if args.src_txt else os.path.join( + model_path, 'alpaca_data.json') + model = LlamaForTextGeneration.from_pretrained(model_path) + + tokenizer = LlamaTokenizer.from_pretrained( + model_path, + model_max_length=512, + padding_side='right', + ) + + special_tokens_dict = dict() + special_tokens_dict['pad_token'] = DEFAULT_PAD_TOKEN + special_tokens_dict['eos_token'] = DEFAULT_EOS_TOKEN + special_tokens_dict['bos_token'] = DEFAULT_BOS_TOKEN + special_tokens_dict['unk_token'] = DEFAULT_UNK_TOKEN + + smart_tokenizer_and_embedding_resize( + special_tokens_dict=special_tokens_dict, + tokenizer=tokenizer, + model=model, + ) + + train_dataset = SupervisedDataset(tokenizer=tokenizer, data_path=data_path) + data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer) + + kwargs = dict( + model=model, + cfg_file=os.path.join(model_path, 'configuration.json'), + train_dataset=train_dataset, + data_collator=data_collator, + max_epochs=3, + work_dir=args.work_dir, + cfg_modify_fn=cfg_modify_fn) + + # Construct trainer and train + trainer = build_trainer( + name=Trainers.text_generation_trainer, default_args=kwargs) + trainer.train() diff --git a/examples/pytorch/llama/run_train_llama.sh b/examples/pytorch/llama/run_train_llama.sh new file mode 100644 index 00000000..7c860d57 --- /dev/null +++ b/examples/pytorch/llama/run_train_llama.sh @@ -0,0 +1,9 @@ +DATA_PARALLEL_SIZE=4 + + +export PYTHONPATH=$PYTHONPATH:./ +torchrun --nproc_per_node $DATA_PARALLEL_SIZE examples/pytorch/llama/finetune_llama.py \ + --work_dir './tmp' \ + --model 'skyline2006/llama-7b' \ + --deepspeed 'default_offload_opt_param.json' \ + --eval_interval 100 diff --git a/modelscope/trainers/hooks/distributed/deepspeed_hook.py b/modelscope/trainers/hooks/distributed/deepspeed_hook.py index d0a6eb9b..3626a9e3 100644 --- a/modelscope/trainers/hooks/distributed/deepspeed_hook.py +++ b/modelscope/trainers/hooks/distributed/deepspeed_hook.py @@ -1,24 +1,131 @@ # Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2020 The HuggingFace Team. All rights reserved. +import math import os import shutil +from functools import partialmethod import deepspeed import torch from deepspeed import DeepSpeedEngine from megatron_util import mpu, print_rank_0 +from transformers.deepspeed import HfTrainerDeepSpeedConfig from modelscope.metainfo import Hooks from modelscope.trainers.hooks import LoadCheckpointHook from modelscope.trainers.hooks.builder import HOOKS from modelscope.trainers.hooks.checkpoint.checkpoint_hook import ( BestCkptSaverHook, CheckpointHook) +from modelscope.trainers.hooks.checkpoint.checkpoint_processor import \ + CheckpointProcessor from modelscope.trainers.hooks.hook import Hook +from modelscope.trainers.hooks.lr_scheduler_hook import (LrSchedulerHook, + LrSchedulerProcessor) +from modelscope.trainers.hooks.optimizer.base import (OptimizerHook, + OptimizerProcessor) from modelscope.trainers.hooks.priority import Priority from modelscope.utils.checkpoint import save_checkpoint +from modelscope.utils.constant import DistributedParallelType +from modelscope.utils.device import create_device from modelscope.utils.logger import get_logger -from ..checkpoint.checkpoint_processor import CheckpointProcessor -from ..lr_scheduler_hook import LrSchedulerHook, LrSchedulerProcessor -from ..optimizer.base import OptimizerHook, OptimizerProcessor +from modelscope.utils.torch_utils import (get_dist_info, get_local_rank, + init_dist) + + +class DeepSpeedConfig(HfTrainerDeepSpeedConfig): + """ + The `DeepSpeedConfig` object is meant to be created during `TrainingArguments` object creation and has the + same lifespan as the latter. + """ + + def is_auto(self, ds_key_long): + val = self.get_value(ds_key_long) + if val is None: + return False + else: + return val == 'auto' + + def trainer_config_finalize(self, args, model, num_training_steps): + """ + This stage runs after we have the model and know num_training_steps. + + Now we can complete the configuration process. + """ + # zero + + # deal with config keys that use `auto` value and rely on model's hidden_size + hidden_size_based_keys = [ + 'zero_optimization.reduce_bucket_size', + 'zero_optimization.stage3_prefetch_bucket_size', + 'zero_optimization.stage3_param_persistence_threshold', + ] + hidden_size_auto_keys = [ + x for x in hidden_size_based_keys if self.is_auto(x) + ] + + if len(hidden_size_auto_keys) > 0: + if hasattr(model.config, 'hidden_size'): + hidden_size = model.config.hidden_size + elif hasattr(model.config, 'hidden_sizes'): + # if there are many hidden sizes pick the largest one + hidden_size = max(model.config.hidden_sizes) + else: + raise ValueError( + "The model's config file has neither `hidden_size` nor `hidden_sizes` entry, " + "therefore it's not possible to automatically fill out the following `auto` entries " + f'in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing ' + '`auto` values for these keys with an integer value of your choice.' + ) + + self.fill_only('zero_optimization.reduce_bucket_size', + hidden_size * hidden_size) + if self.is_zero3(): + # automatically assign the optimal config values based on model config + self.fill_only('zero_optimization.stage3_prefetch_bucket_size', + 0.9 * hidden_size * hidden_size) + self.fill_only( + 'zero_optimization.stage3_param_persistence_threshold', + 10 * hidden_size) + + # scheduler + options = args.train.optimizer.get('options', {}) + warmup = options.get('warmup', {}) + warmup_steps = warmup.get('warmup_steps', 0) + warmup_ratio = warmup.get('warmup_ratio', 0.0) + warmup_steps = warmup_steps if warmup_steps > 0 else math.ceil( + num_training_steps * warmup_ratio) + self.fill_match('scheduler.params.total_num_steps', num_training_steps) + self.fill_match('scheduler.params.warmup_num_steps', warmup_steps) + + if len(self.mismatches) > 0: + mismatches = '\n'.join(self.mismatches) + raise ValueError( + 'Please correct the following DeepSpeed config values that mismatch TrainingArguments' + f" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to 'auto'." + ) + + +def deepspeed_optim_sched(trainer, hf_deepspeed_config, num_training_steps): + config = hf_deepspeed_config.config + optimizer = None + if 'optimizer' not in config: + if hf_deepspeed_config.is_offload(): + logger.info( + 'Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the' + ' custom optimizer has both CPU and GPU implementation (except LAMB)' + ) + + # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch. + # But trainer uses AdamW by default. + optimizer = trainer.optimizer + # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer` + config['zero_allow_untested_optimizer'] = True + + lr_scheduler = None + if 'scheduler' not in config: + lr_scheduler = trainer.scheduler + + return optimizer, lr_scheduler class DeepspeedProcessor(CheckpointProcessor, LrSchedulerProcessor, @@ -58,6 +165,8 @@ class DeepspeedProcessor(CheckpointProcessor, LrSchedulerProcessor, prefix = os.path.basename(checkpoint_path_prefix) trainer.model.save_checkpoint(save_dir, prefix) + if not self.stage3_gather_16bit_weights_on_model_save: + return bin_file = self.get_bin_file() src_file = os.path.join(checkpoint_path_prefix, bin_file) dest_file = os.path.join(output_dir, self._BIN_FILE_DIR, bin_file) @@ -129,6 +238,8 @@ class DeepspeedProcessor(CheckpointProcessor, LrSchedulerProcessor, trainer.model.backward(loss) # update parameters + # Optimizer step for deepspeed must be called on every step regardless of + # the value of gradient accumulation iters trainer.model.step() def initialize_optimizer(self, trainer): @@ -137,20 +248,36 @@ class DeepspeedProcessor(CheckpointProcessor, LrSchedulerProcessor, def step(self, trainer): pass + def should_save_on_rank(self, trainer): + return True + + def get_current_lr(self, trainer): + if isinstance(trainer.optimizer, torch.optim.Optimizer) or isinstance( + trainer.optimizer, deepspeed.DeepSpeedOptimizer): + lr = [group['lr'] for group in trainer.optimizer.param_groups] + elif isinstance(trainer.optimizer, dict): + lr = dict() + for name, optim in trainer.optimizer.items(): + lr[name] = [group['lr'] for group in optim.param_groups] + else: + raise RuntimeError( + 'lr is not applicable because optimizer does not exist.') + return lr + @HOOKS.register_module(module_name=Hooks.DeepspeedHook) class DeepspeedHook(Hook): PRIORITY = Priority.VERY_HIGH def __init__(self, + config=None, deepspeed_activation_checkpointing=True, save_zero_checkpoint=False, with_mpu=True): self.save_zero_checkpoint = save_zero_checkpoint self.deepspeed_activation_checkpointing = deepspeed_activation_checkpointing - # TODO without mpu self.with_mpu = with_mpu - assert with_mpu, 'DeepspeedHook now is only for mpu models.' + self.deepspeed_config = config def register_processor(self, trainer): processor = DeepspeedProcessor() @@ -158,23 +285,76 @@ class DeepspeedHook(Hook): if len(optimizer_hook) > 0 and not isinstance( optimizer_hook[0].processor, DeepspeedProcessor): optimizer_hook[0].set_processor(processor) - lr_schedular_hook = trainer.get_hook(LrSchedulerHook) - if len(lr_schedular_hook) > 0 and not isinstance( - lr_schedular_hook[0].processor, DeepspeedProcessor): - lr_schedular_hook[0].set_processor(processor) ckpt_hook = trainer.get_hook(CheckpointHook) if len(ckpt_hook) > 0 and not isinstance(ckpt_hook[0].processor, DeepspeedProcessor): ckpt_hook[0].set_processor(processor) + best_ckpt_hook = trainer.get_hook(BestCkptSaverHook) if len(best_ckpt_hook) > 0 and not isinstance( best_ckpt_hook[0].processor, DeepspeedProcessor): best_ckpt_hook[0].set_processor(processor) + load_ckpt_hook = trainer.get_hook(LoadCheckpointHook) if len(load_ckpt_hook) > 0 and not isinstance( load_ckpt_hook[0].processor, DeepspeedProcessor): load_ckpt_hook[0].set_processor(processor) + lr_scheduler_hook = trainer.get_hook(LrSchedulerHook) + if len(lr_scheduler_hook) > 0 and not isinstance( + lr_scheduler_hook[0].processor, DeepspeedProcessor): + lr_scheduler_hook[0].set_processor(processor) + self.processor = processor + + def prepare_args(self, args): + args.per_device_train_batch_size = args.train.dataloader.get( + 'batch_size_per_gpu', 4) + args.max_grad_norm = args.train.get('clip_grad', 1.0) + args.learning_rate = args.train.optimizer.get('lr', 2e-5) + args.adam_beta1 = args.train.optimizer.get('adam_beta1', 0.9) + args.adam_beta2 = args.train.optimizer.get('adam_beta2', 0.999) + args.adam_epsilon = args.train.optimizer.get('adam_epsilon', 1e-8) + args.weight_decay = args.train.optimizer.get('weight_decay', 0.0) + args.fp16 = args.train.get('use_fp16', False) + args.fp16_full_eval = args.train.get('use_fp16', False) + args.fp16_backend = args.train.get('fp16_backend', 'amp') + args.save_on_each_node = args.train.get('save_on_each_node', False) + args.fp16_opt_level = args.train.get('fp16_opt_level', None) + args.fp16_opt_level = next((item.get('opt_level', args.fp16_opt_level) + for item in args.train.hooks + if item['type'] == 'ApexAMPOptimizerHook'), + args.fp16_opt_level) + if not args.fp16_opt_level: + args.fp16_opt_level = 'O1' + args.bf16 = args.train.get('bf16', False) + + def get_deepspeed_config(self, trainer, args, max_steps): + _, args.world_size = get_dist_info() + self.prepare_args(args) + if os.path.exists(self.deepspeed_config): + deepspeed_config = self.deepspeed_config + else: + deepspeed_config = os.path.join(trainer.model_dir, + self.deepspeed_config) + if not os.path.exists(deepspeed_config): + raise RuntimeError( + f'No such DeepSpeed json config file: {self.deepspeed_config}.' + ) + self.logger.info(f'Loading deepspeed config from {deepspeed_config}') + + ds_config = DeepSpeedConfig(deepspeed_config) + ds_config.trainer_config_process(args) + + ds_config.trainer_config_finalize(args, trainer.model, max_steps) + return ds_config + + def after_init(self, trainer): + init_dist('pytorch') + local_rank = get_local_rank() + trainer.device = create_device(f'cuda:{local_rank}') + trainer.model.to(trainer.device) + trainer.parallel_groups[DistributedParallelType.DP] = None + def before_val(self, trainer): pass @@ -185,26 +365,23 @@ class DeepspeedHook(Hook): self.logger = trainer.logger # deepspeed init - args = trainer.cfg.train - args.deepspeed_config = os.path.join(trainer.model_dir, - args.deepspeed_config) + args = trainer.cfg + args.gradient_accumulation_steps = args.train.optimizer.get( + 'options', {}).get('cumulative_iters', 1) + num_update_steps_per_epoch = trainer.iters_per_epoch // args.gradient_accumulation_steps + max_steps = math.ceil(trainer._max_epochs * num_update_steps_per_epoch) - trainer.model, _, _, _ = deepspeed.initialize( + ds_config = self.get_deepspeed_config(trainer, args, max_steps) + + optimizer, lr_scheduler = deepspeed_optim_sched( + trainer, ds_config, max_steps) + config = ds_config.config + self.processor.stage3_gather_16bit_weights_on_model_save = config[ + 'zero_optimization'].get( + 'stage3_gather_16bit_weights_on_model_save', True) + + trainer.model, trainer.optimizer, _, trainer.lr_scheduler = deepspeed.initialize( model=trainer.model, - optimizer=trainer.optimizer, - args=args, - lr_scheduler=trainer.lr_scheduler, - mpu=mpu, - dist_init_required=False) - trainer.model.save_zero_checkpoint = self.save_zero_checkpoint - - if self.deepspeed_activation_checkpointing: - model = trainer.unwrap_module(trainer.model) - deepspeed.checkpointing.configure( - mpu, - deepspeed_config=args.deepspeed_config, - num_checkpoints=model.config.num_hidden_layers) - - mpu.checkpoint = deepspeed.checkpointing.checkpoint - mpu.get_cuda_rng_tracker = deepspeed.checkpointing.get_cuda_rng_tracker - mpu.model_parallel_cuda_manual_seed = deepspeed.checkpointing.model_parallel_cuda_manual_seed + optimizer=optimizer, + config=config, + lr_scheduler=lr_scheduler) diff --git a/modelscope/trainers/hooks/lr_scheduler_hook.py b/modelscope/trainers/hooks/lr_scheduler_hook.py index 51a8e858..facf5155 100644 --- a/modelscope/trainers/hooks/lr_scheduler_hook.py +++ b/modelscope/trainers/hooks/lr_scheduler_hook.py @@ -39,6 +39,20 @@ class LrSchedulerProcessor: else: trainer.lr_scheduler.step() + def get_current_lr(self, trainer): + import torch + + if isinstance(trainer.optimizer, torch.optim.Optimizer): + lr = [group['lr'] for group in trainer.optimizer.param_groups] + elif isinstance(trainer.optimizer, dict): + lr = dict() + for name, optim in trainer.optimizer.items(): + lr[name] = [group['lr'] for group in optim.param_groups] + else: + raise RuntimeError( + 'lr is not applicable because optimizer does not exist.') + return lr + class LrStrategy: by_epoch = 'by_epoch' @@ -84,20 +98,6 @@ class LrSchedulerHook(Hook): self.processor.initialize_lr_scheduler(trainer) - def get_current_lr(self, trainer): - import torch - - if isinstance(trainer.optimizer, torch.optim.Optimizer): - lr = [group['lr'] for group in trainer.optimizer.param_groups] - elif isinstance(trainer.optimizer, dict): - lr = dict() - for name, optim in trainer.optimizer.items(): - lr[name] = [group['lr'] for group in optim.param_groups] - else: - raise RuntimeError( - 'lr is not applicable because optimizer does not exist.') - return lr - def after_train_iter(self, trainer): if self.lr_strategy == LrStrategy.by_step and trainer.iter >= getattr( trainer, 'cumulative_iters', 1) - 1: @@ -112,7 +112,7 @@ class LrSchedulerHook(Hook): self.processor.step(trainer) def _get_log_lr(self, trainer): - cur_lr = self.get_current_lr(trainer) + cur_lr = self.processor.get_current_lr(trainer) # only record lr of the first param group if isinstance(cur_lr, list): lr = cur_lr[0] diff --git a/modelscope/trainers/trainer.py b/modelscope/trainers/trainer.py index c980de04..fd0fafb8 100644 --- a/modelscope/trainers/trainer.py +++ b/modelscope/trainers/trainer.py @@ -1001,7 +1001,7 @@ class EpochBasedTrainer(BaseTrainer): """ optimizer, lr_scheduler = self.optimizers if optimizer is None: - optimizer_cfg = self.cfg.train.get('optimizer', None) + optimizer_cfg = deepcopy(self.cfg.train.get('optimizer', None)) else: optimizer_cfg = None @@ -1011,7 +1011,8 @@ class EpochBasedTrainer(BaseTrainer): optimizer = self.build_optimizer(cfg=optimizer_cfg) if lr_scheduler is None: - lr_scheduler_cfg = self.cfg.train.get('lr_scheduler', None) + lr_scheduler_cfg = deepcopy( + self.cfg.train.get('lr_scheduler', None)) else: lr_scheduler_cfg = None From cc3c384d5ed72f4c0c29e245e91b342bbb846234 Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Thu, 15 Jun 2023 15:42:21 +0800 Subject: [PATCH 70/76] Fix issues for downloading mplug-youku dataset 1. Optimize downloading meta-csv files for large-scale dataset like mPLUG-youku (> 1GB for meta csv mapping) 2. Add head and overall progress bar for NativeIterableDataset 3. Modify the try-catch info for oss_utils Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12952842 --- modelscope/hub/api.py | 23 +++---- .../data_files/data_files_manager.py | 1 + modelscope/msdatasets/dataset_cls/dataset.py | 64 +++++++++++++------ .../msdatasets/download/dataset_builder.py | 24 ++++--- .../msdatasets/download/download_config.py | 1 + .../msdatasets/meta/data_meta_manager.py | 3 +- modelscope/msdatasets/ms_dataset.py | 3 +- modelscope/msdatasets/utils/dataset_utils.py | 33 +++++++--- modelscope/msdatasets/utils/oss_utils.py | 3 +- .../test_efficient_diffusion_tuning.py | 2 +- 10 files changed, 103 insertions(+), 54 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 75e02b9d..2f14976d 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -47,7 +47,8 @@ from modelscope.utils.constant import (DEFAULT_DATASET_REVISION, MASTER_MODEL_BRANCH, DatasetFormations, DatasetMetaFormats, DatasetVisibilityMap, DownloadChannel, - ModelFile, VirgoDatasetConfig) + DownloadMode, ModelFile, + VirgoDatasetConfig) from modelscope.utils.logger import get_logger from .utils.utils import (get_endpoint, get_release_datetime, model_id_to_group_owner_name) @@ -641,27 +642,21 @@ class HubApi: return local_paths, dataset_formation - def fetch_single_csv_script(self, script_url: str): - cookies = ModelScopeConfig.get_cookies() - resp = self.session.get(script_url, cookies=cookies, headers=self.headers) - if not resp or not resp.text: - raise 'The meta-csv file cannot be empty when the meta-args `big_data` is true.' - text_list = resp.text.strip().split('\n') - text_headers = text_list[0] - text_content = text_list[1:] - - return text_headers, text_content - @staticmethod - def fetch_csv_from_url(url, out_path, chunk_size=1000000): + def fetch_csv_from_url(url, out_path, chunk_size=100000, mode=DownloadMode.REUSE_DATASET_IF_EXISTS): from io import StringIO import hashlib out_path = os.path.join(out_path, hashlib.md5(url.encode(encoding='UTF-8')).hexdigest()) + if mode == DownloadMode.FORCE_REDOWNLOAD and os.path.exists(out_path): + os.remove(out_path) if os.path.exists(out_path): + logger.info(f'Reusing cached meta-csv file: {out_path}') return out_path cookies = ModelScopeConfig.get_cookies() # Make the request and get the response content as TextIO + logger.info('Loading meta-csv file ...') + response = requests.get(url, cookies=cookies) data = StringIO(response.text) @@ -673,7 +668,7 @@ class HubApi: while loop: try: chunk = csv_file_reader.get_chunk(size=chunk_size) - + logger.info(f'Receiving chunk {iter_num}, shape: {chunk.shape}') if iter_num == 0: with_header = True else: diff --git a/modelscope/msdatasets/data_files/data_files_manager.py b/modelscope/msdatasets/data_files/data_files_manager.py index a84876f6..d5f43533 100644 --- a/modelscope/msdatasets/data_files/data_files_manager.py +++ b/modelscope/msdatasets/data_files/data_files_manager.py @@ -58,6 +58,7 @@ class DataFilesManager(object): # Set context. Note: no need to update context_config. download_config.oss_config = self.oss_config + download_config.num_proc = self.input_config_kwargs.get('num_proc', 4) dataset_context_config.download_config = download_config self.dataset_context_config = dataset_context_config os.makedirs(download_config.cache_dir, exist_ok=True) diff --git a/modelscope/msdatasets/dataset_cls/dataset.py b/modelscope/msdatasets/dataset_cls/dataset.py index 9114285e..cebfcfba 100644 --- a/modelscope/msdatasets/dataset_cls/dataset.py +++ b/modelscope/msdatasets/dataset_cls/dataset.py @@ -6,6 +6,7 @@ import os import datasets import pandas as pd from datasets import IterableDataset +from tqdm import tqdm from modelscope.msdatasets.utils.maxcompute_utils import MaxComputeUtil from modelscope.utils.constant import (DEFAULT_MAXCOMPUTE_ENDPOINT, @@ -23,15 +24,18 @@ class ExternalDataset(object): def __init__(self, split_path_dict, config_kwargs): self.split_path_dict = split_path_dict self.config_kwargs = copy.deepcopy(config_kwargs) - self.config_kwargs.update({'split_config': split_path_dict}) + self.config_kwargs.update({'split_config': self.split_path_dict}) # dataset for specific extensions self.spec_extension_dataset = None - self.split_data_files = {k: [] for k, _ in split_path_dict.items()} + self.split_data_files = { + k: [] + for k, _ in self.split_path_dict.items() + } self.custom_map = {} # the extension of file file_ext = '' - for split_name, split_dir in split_path_dict.items(): + for split_name, split_dir in self.split_path_dict.items(): if isinstance(split_dir, str) and os.path.isdir(split_dir): split_file_names = os.listdir(split_dir) set_files_exts = set([ @@ -91,28 +95,52 @@ class NativeIterableDataset(IterableDataset): super().__init__(ex_iterable=ex_iterable, info=info, split=split) def __iter__(self): - for key, entity in self._iter(): + for key, entity in tqdm( + self._iter(), + desc='Overall progress', + total=self.n_shards, + dynamic_ncols=True): if isinstance(entity, dict): ret = {} - for k, v in entity.items(): - ret[k] = v - if k.endswith(':FILE'): - dl_manager = self._ex_iterable.kwargs.get('dl_manager') - ex_cache_path = dl_manager.download_and_extract(v) - ret[k] = ex_cache_path - if k.endswith('Image:FILE'): - from PIL import Image - ret[k + ':Object'] = Image.open(fp=ex_cache_path) - if k.endswith('Audio:FILE'): - import torchaudio - waveform_and_rate = torchaudio.load(ex_cache_path) - ret[k + ':Object'] = waveform_and_rate + try: + for k, v in entity.items(): + ret[k] = v + if k.endswith(':FILE'): + dl_manager = self._ex_iterable.kwargs.get( + 'dl_manager') + ex_cache_path = dl_manager.download_and_extract(v) + ret[k] = ex_cache_path + if k.endswith('Image:FILE'): + from PIL import Image + ret[k + + ':Object'] = Image.open(fp=ex_cache_path) + if k.endswith('Audio:FILE'): + import torchaudio + waveform_and_rate = torchaudio.load( + ex_cache_path) + ret[k + ':Object'] = waveform_and_rate + except Exception as e: + logger.error(e) + ret = {} + entity = ret yield entity def __len__(self): - return 1 + return self.n_shards + + def head(self, n=5): + """ + Returns the first n rows of the dataset. + + Args: + n (int): Number of rows to return. + + Returns: + Dict[str, list]: e.g. {'col1': [val11, val12, ...], 'col2': [val21, val22, ...]} + """ + return self._head(n=n) class VirgoDataset(object): diff --git a/modelscope/msdatasets/download/dataset_builder.py b/modelscope/msdatasets/download/dataset_builder.py index b3ed5a8e..796e3d83 100644 --- a/modelscope/msdatasets/download/dataset_builder.py +++ b/modelscope/msdatasets/download/dataset_builder.py @@ -325,6 +325,9 @@ class IterableDatasetBuilder(csv.Csv): self.info.builder_name = self.dataset_name self.name = camelcase_to_snakecase(self.dataset_name) + self.meta_csv_df = None + self.meta_cache_dir = dataset_context_config.data_meta_config.meta_cache_dir + @staticmethod def get_builder_instance( dataset_context_config: DatasetContextConfig) -> csv.Csv: @@ -449,17 +452,13 @@ class IterableDatasetBuilder(csv.Csv): zip_file_name = os.path.splitext(zip_file)[0] if meta_file_url and not files: - headers, texts = hub_api.fetch_single_csv_script(meta_file_url) - meta_csv_mapping = IterableDatasetBuilder.trans_data_to_mapping( - headers, texts, self.csv_delimiter) - pa_table = pa.Table.from_pydict(meta_csv_mapping) + self._get_meta_csv_df(meta_file_url) + pa_table = pa.Table.from_pandas(self.meta_csv_df) yield 0, pa_table elif meta_file_url and files: # Get meta file - headers, texts = hub_api.fetch_single_csv_script(meta_file_url) - meta_csv_mapping = IterableDatasetBuilder.trans_data_to_mapping( - headers, texts, self.csv_delimiter) + self._get_meta_csv_df(meta_file_url) if is_zip: oss_config_for_unzipped = hub_api.get_dataset_access_config_for_unzipped( @@ -467,7 +466,7 @@ class IterableDatasetBuilder(csv.Csv): zip_file_name) dl_manager.download_config.oss_config = oss_config_for_unzipped - pa_table = pa.Table.from_pydict(meta_csv_mapping) + pa_table = pa.Table.from_pandas(self.meta_csv_df) yield 0, pa_table elif not meta_file_url and files: @@ -477,6 +476,15 @@ class IterableDatasetBuilder(csv.Csv): else: raise f'Neither column meta nor data file found in {self.dataset_name}.json .' + def _get_meta_csv_df(self, meta_file_url: str) -> None: + if not self.meta_csv_df: + meta_csv_file_path = HubApi.fetch_csv_from_url( + meta_file_url, self.meta_cache_dir) + self.meta_csv_df = pd.read_csv( + meta_csv_file_path, + iterator=False, + delimiter=self.csv_delimiter) + @staticmethod def trans_data_to_mapping(headers: str, texts: list, delimiter: str): res = {} diff --git a/modelscope/msdatasets/download/download_config.py b/modelscope/msdatasets/download/download_config.py index 4af656e0..11118f85 100644 --- a/modelscope/msdatasets/download/download_config.py +++ b/modelscope/msdatasets/download/download_config.py @@ -15,6 +15,7 @@ class DataDownloadConfig(DownloadConfig): self.data_dir: Optional[str] = None self.oss_config: Optional[dict] = {} self.meta_args_map: Optional[dict] = {} + self.num_proc: int = 4 def copy(self) -> 'DataDownloadConfig': return self diff --git a/modelscope/msdatasets/meta/data_meta_manager.py b/modelscope/msdatasets/meta/data_meta_manager.py index 0fa74c37..3f1e6572 100644 --- a/modelscope/msdatasets/meta/data_meta_manager.py +++ b/modelscope/msdatasets/meta/data_meta_manager.py @@ -130,7 +130,8 @@ class DataMetaManager(object): target_subset_name, target_dataset_structure = get_target_dataset_structure( dataset_json, subset_name, split) meta_map, file_map, args_map, type_map = get_dataset_files( - target_dataset_structure, dataset_name, namespace, version) + target_dataset_structure, dataset_name, namespace, + self.dataset_context_config, version) data_meta_config.meta_data_files = meta_map data_meta_config.zip_data_files = file_map diff --git a/modelscope/msdatasets/ms_dataset.py b/modelscope/msdatasets/ms_dataset.py index fa5bac8d..dea65d56 100644 --- a/modelscope/msdatasets/ms_dataset.py +++ b/modelscope/msdatasets/ms_dataset.py @@ -60,7 +60,8 @@ class MsDataset: _dataset_context_config: DatasetContextConfig = None def __init__(self, - ds_instance: Union[Dataset, IterableDataset, ExternalDataset], + ds_instance: Union[Dataset, IterableDataset, ExternalDataset, + NativeIterableDataset], target: Optional[str] = None): self._hf_ds = ds_instance if target is not None and target not in self._hf_ds.features: diff --git a/modelscope/msdatasets/utils/dataset_utils.py b/modelscope/msdatasets/utils/dataset_utils.py index dde044d5..dc7f9b38 100644 --- a/modelscope/msdatasets/utils/dataset_utils.py +++ b/modelscope/msdatasets/utils/dataset_utils.py @@ -4,7 +4,11 @@ import os from collections import defaultdict from typing import Optional, Union +import pandas as pd + from modelscope.hub.api import HubApi +from modelscope.msdatasets.context.dataset_context_config import \ + DatasetContextConfig from modelscope.utils.constant import DEFAULT_DATASET_REVISION, MetaDataFields from modelscope.utils.logger import get_logger @@ -169,6 +173,7 @@ def get_split_objects_map(file_map, objects): def get_dataset_files(subset_split_into: dict, dataset_name: str, namespace: str, + context_config: DatasetContextConfig, revision: Optional[str] = DEFAULT_DATASET_REVISION): """ Return: @@ -186,6 +191,7 @@ def get_dataset_files(subset_split_into: dict, args_map = defaultdict(dict) custom_type_map = defaultdict(dict) modelscope_api = HubApi() + meta_cache_dir = context_config.data_meta_config.meta_cache_dir for split, info in subset_split_into.items(): custom_type_map[split] = info.get('custom', '') @@ -200,16 +206,23 @@ def get_dataset_files(subset_split_into: dict, for split, args_dict in args_map.items(): if args_dict and args_dict.get(MetaDataFields.ARGS_BIG_DATA): meta_csv_file_url = meta_map[split] - _, script_content = modelscope_api.fetch_single_csv_script( - meta_csv_file_url) - if not script_content: - raise 'Meta-csv file cannot be empty when meta-args `big_data` is true.' - for item in script_content: - if not item: - continue - item = item.strip().split(',')[0] - if item: - objects.append(item) + + meta_csv_file_path = HubApi.fetch_csv_from_url( + meta_csv_file_url, meta_cache_dir) + + csv_delimiter = context_config.config_kwargs.get('delimiter', ',') + csv_df = pd.read_csv( + meta_csv_file_path, iterator=False, delimiter=csv_delimiter) + target_col = csv_df.columns[csv_df.columns.str.contains( + ':FILE')].to_list() + if len(target_col) == 0: + logger.error( + f'No column contains ":FILE" in {meta_csv_file_path}.') + target_col = csv_df.columns[0] + else: + target_col = target_col[0] + objects = csv_df[target_col].to_list() + file_map[split] = objects # More general but low-efficiency. if not objects: diff --git a/modelscope/msdatasets/utils/oss_utils.py b/modelscope/msdatasets/utils/oss_utils.py index 44ee5446..d11b5035 100644 --- a/modelscope/msdatasets/utils/oss_utils.py +++ b/modelscope/msdatasets/utils/oss_utils.py @@ -117,7 +117,8 @@ class OssUtilities: if e.__dict__.get('status') == 403: self._reload_sts() if retry_count >= self.max_retries: - raise + logger.warning(f'Failed to download {oss_file_name}') + raise e return local_path diff --git a/tests/pipelines/test_efficient_diffusion_tuning.py b/tests/pipelines/test_efficient_diffusion_tuning.py index e33b2bf2..f1aa52de 100644 --- a/tests/pipelines/test_efficient_diffusion_tuning.py +++ b/tests/pipelines/test_efficient_diffusion_tuning.py @@ -13,7 +13,7 @@ class EfficientDiffusionTuningTest(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.efficient_diffusion_tuning - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_efficient_diffusion_tuning_lora_run_pipeline(self): model_id = 'damo/multi-modal_efficient-diffusion-tuning-lora' inputs = {'prompt': 'pale golden rod circle with old lace background'} From ad5d1aeb624aad3701fe3ccb5c699e41b033f24f Mon Sep 17 00:00:00 2001 From: "hemu.zp" Date: Mon, 19 Jun 2023 11:25:27 +0800 Subject: [PATCH 71/76] Fix gpt3 finetune nan Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12981998 * fix gpt3 finetune nan --- modelscope/models/nlp/gpt3/distributed_gpt3.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modelscope/models/nlp/gpt3/distributed_gpt3.py b/modelscope/models/nlp/gpt3/distributed_gpt3.py index 570487df..c7917b31 100644 --- a/modelscope/models/nlp/gpt3/distributed_gpt3.py +++ b/modelscope/models/nlp/gpt3/distributed_gpt3.py @@ -1023,7 +1023,11 @@ class DistributedGPT3(TorchModel, StreamingOutputMixin): losses = losses.float() loss_mask = loss_mask.view(-1).float() - loss = torch.sum(losses.view(-1) * loss_mask) / loss_mask.sum() + mask_sum = loss_mask.sum() + if mask_sum == 0: + loss = torch.sum(losses.view(-1)).zero_() + else: + loss = torch.sum(losses.view(-1) * loss_mask) / mask_sum return TextGenerationModelOutput(logits=logits, loss=loss) From fa7562fd96a04a61404368449bf1ae3c8e5e7801 Mon Sep 17 00:00:00 2001 From: "mulin.lyh" Date: Mon, 19 Jun 2023 17:56:04 +0800 Subject: [PATCH 72/76] [to #50384694]fix: ci hang when build torch extension Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12976382 --- .dev_scripts/ci_container_test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.dev_scripts/ci_container_test.sh b/.dev_scripts/ci_container_test.sh index 8d1c9a0c..931ca397 100644 --- a/.dev_scripts/ci_container_test.sh +++ b/.dev_scripts/ci_container_test.sh @@ -32,6 +32,8 @@ if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then else echo "Running case in release image, run case directly!" fi +# remove torch_extensions folder to avoid ci hang. +rm -rf ~/.cache/torch_extensions if [ $# -eq 0 ]; then ci_command="python tests/run.py --subprocess" else From bc5c16aa10607ed7df6d1e737b01b0ef968be720 Mon Sep 17 00:00:00 2001 From: "lanjinpeng.ljp" Date: Tue, 20 Jun 2023 19:25:24 +0800 Subject: [PATCH 73/76] support third_party key in pipeline Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12940228 * support third_party key in pipeline --- modelscope/pipelines/builder.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/modelscope/pipelines/builder.py b/modelscope/pipelines/builder.py index dd39453c..ca1431ea 100644 --- a/modelscope/pipelines/builder.py +++ b/modelscope/pipelines/builder.py @@ -7,7 +7,8 @@ from modelscope.hub.snapshot_download import snapshot_download from modelscope.metainfo import DEFAULT_MODEL_FOR_PIPELINE, Pipelines from modelscope.models.base import Model from modelscope.utils.config import ConfigDict, check_config -from modelscope.utils.constant import DEFAULT_MODEL_REVISION, Invoke +from modelscope.utils.constant import (DEFAULT_MODEL_REVISION, Invoke, + ThirdParty) from modelscope.utils.hub import read_config from modelscope.utils.plugins import (register_modelhub_repo, register_plugins_repo) @@ -18,7 +19,7 @@ from .util import is_official_hub_path PIPELINES = Registry('pipelines') -def normalize_model_input(model, model_revision): +def normalize_model_input(model, model_revision, third_party=None): """ normalize the input model, to ensure that a model str is a valid local path: in other words, for model represented by a model id, the model shall be downloaded locally """ @@ -26,19 +27,21 @@ def normalize_model_input(model, model_revision): # skip revision download if model is a local directory if not os.path.exists(model): # note that if there is already a local copy, snapshot_download will check and skip downloading + user_agent = {Invoke.KEY: Invoke.PIPELINE} + if third_party is not None: + user_agent[ThirdParty.KEY] = third_party model = snapshot_download( - model, - revision=model_revision, - user_agent={Invoke.KEY: Invoke.PIPELINE}) + model, revision=model_revision, user_agent=user_agent) elif isinstance(model, list) and isinstance(model[0], str): for idx in range(len(model)): if is_official_hub_path( model[idx], model_revision) and not os.path.exists(model[idx]): + user_agent = {Invoke.KEY: Invoke.PIPELINE} + if third_party is not None: + user_agent[ThirdParty.KEY] = third_party model[idx] = snapshot_download( - model[idx], - revision=model_revision, - user_agent={Invoke.KEY: Invoke.PIPELINE}) + model[idx], revision=model_revision, user_agent=user_agent) return model @@ -97,7 +100,11 @@ def pipeline(task: str = None, if task is None and pipeline_name is None: raise ValueError('task or pipeline_name is required') - model = normalize_model_input(model, model_revision) + third_party = kwargs.get(ThirdParty.KEY) + if third_party is not None: + kwargs.pop(ThirdParty.KEY) + model = normalize_model_input( + model, model_revision, third_party=third_party) pipeline_props = {'type': pipeline_name} if pipeline_name is None: # get default pipeline for this task From 0db0ec558634f113e5168d826f1a820f327824e1 Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Wed, 21 Jun 2023 10:22:06 +0800 Subject: [PATCH 74/76] Merge code from github 1. Merge(add) daily regression from github PR (daily_regression.yaml) 2. Add lora stable diffusion from github PR Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/13010802 * fix: device arg not work, rename device to ngpu (#272) * Correcting the lora stable diffusion example script (#300) * add vad model and punc model in README.md add vad model and punc model * Merge pull request #302 from modelscope/langgz-patch-1 add vad model and punc model in README.md * add 1.6 * modify ignore * Merge pull request #307 from modelscope/dev_rs_16 Merge release 1.6 * undo datetime to 2099 * Merge pull request #311 from modelscope/fix_master_version undo datetime to 2099 * add daily regression workflow * modify workflow name * fix cron format issue * lora trainer * Merge pull request #315 from liuyhwangyh/add_regression_workflow add daily regression workflow --- .github/workflows/daily_regression.yaml | 38 +++++ .../api/modelscope.pipelines.multi_modal.rst | 2 +- .../finetune_stable_diffusion.py | 29 ++-- .../pytorch/stable_diffusion/run_train.sh | 10 +- modelscope/metainfo.py | 3 + .../multi_modal/stable_diffusion/__init__.py | 2 + .../stable_diffusion/stable_diffusion.py | 152 ++++++++++++++++++ modelscope/msdatasets/ms_dataset.py | 2 +- modelscope/pipelines/multi_modal/__init__.py | 4 +- .../multi_modal/diffusers_wrapped/__init__.py | 4 +- .../stable_diffusion/__init__.py | 4 +- .../stable_diffusion_pipeline.py | 57 +++---- modelscope/preprocessors/multi_modal.py | 5 +- .../multi_modal/lora_diffusion/__init__.py | 1 + .../lora_diffusion/lora_diffusion_trainer.py | 78 +++++++++ .../multi_modal/stable_diffusion/__init__.py | 3 + .../stable_diffusion_trainer.py | 30 ++++ modelscope/tuners/control_sd_lora.py | 18 ++- modelscope/tuners/sd_lora.py | 7 +- modelscope/version.py | 2 +- requirements/multi-modal.txt | 6 +- .../test_diffusers_stable_diffusion.py | 4 +- ...test_efficient_diffusion_tuning_trainer.py | 8 +- tests/trainers/test_lora_diffusion_trainer.py | 89 ++++++++++ .../trainers/test_stable_diffusion_trainer.py | 89 ++++++++++ 25 files changed, 573 insertions(+), 74 deletions(-) create mode 100644 .github/workflows/daily_regression.yaml create mode 100644 modelscope/models/multi_modal/stable_diffusion/__init__.py create mode 100644 modelscope/models/multi_modal/stable_diffusion/stable_diffusion.py create mode 100644 modelscope/trainers/multi_modal/lora_diffusion/__init__.py create mode 100644 modelscope/trainers/multi_modal/lora_diffusion/lora_diffusion_trainer.py create mode 100644 modelscope/trainers/multi_modal/stable_diffusion/__init__.py create mode 100644 modelscope/trainers/multi_modal/stable_diffusion/stable_diffusion_trainer.py create mode 100644 tests/trainers/test_lora_diffusion_trainer.py create mode 100644 tests/trainers/test_stable_diffusion_trainer.py diff --git a/.github/workflows/daily_regression.yaml b/.github/workflows/daily_regression.yaml new file mode 100644 index 00000000..0500b61c --- /dev/null +++ b/.github/workflows/daily_regression.yaml @@ -0,0 +1,38 @@ +name: daily_regression + +on: + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 0 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + unittest: + # The type of runner that the job will run on + runs-on: [modelscope-self-hosted] + steps: + - name: ResetFileMode + shell: bash + run: | + # reset filemode to allow action runner to delete files + # generated by root in docker + set -e + source ~/.bashrc + sudo chown -R $USER:$USER $ACTION_RUNNER_DIR + + - name: Checkout + uses: actions/checkout@v2 + with: + lfs: 'true' + submodules: 'true' + - name: Checkout LFS objects + run: git lfs checkout + - name: Run unittest + shell: bash + run: | + set -e + source /mnt/modelscope/regression_env.sh + bash .dev_scripts/dockerci.sh diff --git a/docs/source/api/modelscope.pipelines.multi_modal.rst b/docs/source/api/modelscope.pipelines.multi_modal.rst index b62878e3..ffb65fad 100644 --- a/docs/source/api/modelscope.pipelines.multi_modal.rst +++ b/docs/source/api/modelscope.pipelines.multi_modal.rst @@ -18,7 +18,7 @@ modelscope.pipelines.multi_modal ImageCaptioningPipeline MGeoRankingPipeline MultiModalEmbeddingPipeline - StableDiffusionWrapperPipeline + StableDiffusionPipeline TextToImageSynthesisPipeline VideoCaptioningPipeline VideoMultiModalEmbeddingPipeline diff --git a/examples/pytorch/stable_diffusion/finetune_stable_diffusion.py b/examples/pytorch/stable_diffusion/finetune_stable_diffusion.py index 28ba853c..ac16ed2c 100644 --- a/examples/pytorch/stable_diffusion/finetune_stable_diffusion.py +++ b/examples/pytorch/stable_diffusion/finetune_stable_diffusion.py @@ -1,17 +1,21 @@ -from dataclasses import dataclass, field - +from modelscope.metainfo import Trainers from modelscope.msdatasets import MsDataset from modelscope.trainers import EpochBasedTrainer, build_trainer from modelscope.trainers.training_args import TrainingArgs +from modelscope.utils.constant import DownloadMode -training_args = TrainingArgs(task='efficient-diffusion-tuning').parse_cli() +training_args = TrainingArgs(task='text-to-image-synthesis').parse_cli() config, args = training_args.to_config() print(args) -dataset = MsDataset.load( - args.train_dataset_name, namespace=args.train_dataset_namespace) -train_dataset = dataset['train'] -validation_dataset = dataset['validation'] +train_dataset = MsDataset.load( + args.train_dataset_name, + split='train', + download_mode=DownloadMode.FORCE_REDOWNLOAD) +validation_dataset = MsDataset.load( + args.train_dataset_name, + split='validation', + download_mode=DownloadMode.FORCE_REDOWNLOAD) def cfg_modify_fn(cfg): @@ -19,17 +23,22 @@ def cfg_modify_fn(cfg): cfg.merge_from_dict(config) else: cfg = config - cfg.train.lr_scheduler.T_max = training_args.max_epochs - cfg.model.inference = False + cfg.train.lr_scheduler = { + 'type': 'LambdaLR', + 'lr_lambda': lambda _: 1, + 'last_epoch': -1 + } + cfg.train.optimizer.lr = 1e-4 return cfg kwargs = dict( model=training_args.model, + model_revision='v1.0.6', work_dir=training_args.work_dir, train_dataset=train_dataset, eval_dataset=validation_dataset, cfg_modify_fn=cfg_modify_fn) -trainer: EpochBasedTrainer = build_trainer(name='trainer', default_args=kwargs) +trainer = build_trainer(name=Trainers.lora_diffusion, default_args=kwargs) trainer.train() diff --git a/examples/pytorch/stable_diffusion/run_train.sh b/examples/pytorch/stable_diffusion/run_train.sh index 033c1636..8e45ae88 100644 --- a/examples/pytorch/stable_diffusion/run_train.sh +++ b/examples/pytorch/stable_diffusion/run_train.sh @@ -1,9 +1,9 @@ PYTHONPATH=. torchrun examples/pytorch/stable_diffusion/finetune_stable_diffusion.py \ - --model 'damo/multi-modal_efficient-diffusion-tuning-lora' \ - --work_dir './tmp/stable_diffusion_tuning' \ - --train_dataset_namespace 'damo' \ - --train_dataset_name 'buptwq/lora-stable-diffusion-finetune-dog' \ - --max_epochs 150 \ + --model 'AI-ModelScope/stable-diffusion-v1-5' \ + --model_revision 'v1.0.6' \ + --work_dir './tmp/lora_diffusion' \ + --train_dataset_name 'buptwq/lora-stable-diffusion-finetune' \ + --max_epochs 100 \ --save_ckpt_strategy 'by_epoch' \ --logging_interval 100 \ --train.dataloader.workers_per_gpu 0 \ diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index c4057314..f2529be2 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -205,6 +205,7 @@ class Models(object): efficient_diffusion_tuning = 'efficient-diffusion-tuning' mplug_owl = 'mplug-owl' clip_interrogator = 'clip-interrogator' + stable_diffusion = 'stable-diffusion' # science models unifold = 'unifold' @@ -892,6 +893,8 @@ class MultiModalTrainers(object): mplug = 'mplug' mgeo_ranking_trainer = 'mgeo-ranking-trainer' efficient_diffusion_tuning = 'efficient-diffusion-tuning' + stable_diffusion = 'stable-diffusion' + lora_diffusion = 'lora-diffusion' class AudioTrainers(object): diff --git a/modelscope/models/multi_modal/stable_diffusion/__init__.py b/modelscope/models/multi_modal/stable_diffusion/__init__.py new file mode 100644 index 00000000..9ffff12a --- /dev/null +++ b/modelscope/models/multi_modal/stable_diffusion/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .stable_diffusion import StableDiffusion diff --git a/modelscope/models/multi_modal/stable_diffusion/stable_diffusion.py b/modelscope/models/multi_modal/stable_diffusion/stable_diffusion.py new file mode 100644 index 00000000..8ec2149d --- /dev/null +++ b/modelscope/models/multi_modal/stable_diffusion/stable_diffusion.py @@ -0,0 +1,152 @@ +# Copyright 2023-2024 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import os +from functools import partial +from typing import Callable, List, Optional, Union + +import torch +import torch.nn.functional as F +from diffusers import AutoencoderKL, DDPMScheduler, UNet2DConditionModel +from transformers import CLIPTextModel, CLIPTokenizer + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.checkpoint import save_checkpoint, save_configuration +from modelscope.utils.constant import Tasks + + +@MODELS.register_module( + Tasks.text_to_image_synthesis, module_name=Models.stable_diffusion) +class StableDiffusion(TorchModel): + """ The implementation of efficient diffusion tuning model based on TorchModel. + + This model is constructed with the implementation of stable diffusion model. If you want to + finetune lightweight parameters on your own dataset, you can define you own tuner module + and load in this cls. + """ + + def __init__(self, model_dir, *args, **kwargs): + """ Initialize a vision efficient diffusion tuning model. + + Args: + model_dir: model id or path, where model_dir/pytorch_model.bin + """ + super().__init__(model_dir, *args, **kwargs) + pretrained_model_name_or_path = kwargs.pop( + 'pretrained_model_name_or_path', 'runwayml/stable-diffusion-v1-5') + revision = kwargs.pop('revision', None) + self.lora_tune = kwargs.pop('lora_tune', True) + + self.weight_dtype = torch.float32 + self.device = torch.device( + 'cuda' if torch.cuda.is_available() else 'cpu') + + # Load scheduler, tokenizer and models + self.noise_scheduler = DDPMScheduler.from_pretrained( + pretrained_model_name_or_path, subfolder='scheduler') + self.tokenizer = CLIPTokenizer.from_pretrained( + pretrained_model_name_or_path, + subfolder='tokenizer', + revision=revision) + self.text_encoder = CLIPTextModel.from_pretrained( + pretrained_model_name_or_path, + subfolder='text_encoder', + revision=revision) + self.vae = AutoencoderKL.from_pretrained( + pretrained_model_name_or_path, subfolder='vae', revision=revision) + self.unet = UNet2DConditionModel.from_pretrained( + pretrained_model_name_or_path, subfolder='unet', revision=revision) + + # Freeze gradient calculation and move to device + if self.vae is not None: + self.vae.requires_grad_(False) + self.vae = self.vae.to(self.device) + if self.text_encoder is not None: + self.text_encoder.requires_grad_(False) + self.text_encoder = self.text_encoder.to(self.device) + if self.unet is not None: + if self.lora_tune: + self.unet.requires_grad_(False) + self.unet = self.unet.to(self.device) + + def tokenize_caption(self, captions): + """ Convert caption text to token data. + + Args: + captions: a batch of texts. + Returns: token's data as tensor. + """ + inputs = self.tokenizer( + captions, + max_length=self.tokenizer.model_max_length, + padding='max_length', + truncation=True, + return_tensors='pt') + return inputs.input_ids + + def forward(self, text='', target=None): + self.unet.train() + self.unet = self.unet.to(self.device) + + with torch.no_grad(): + latents = self.vae.encode( + target.to(dtype=self.weight_dtype)).latent_dist.sample() + latents = latents * self.vae.config.scaling_factor + + # Sample noise that we'll add to the latents + noise = torch.randn_like(latents) + bsz = latents.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint( + 0, + self.noise_scheduler.num_train_timesteps, (bsz, ), + device=latents.device) + timesteps = timesteps.long() + + # Add noise to the latents according to the noise magnitude at each timestep + # (this is the forward diffusion process) + noisy_latents = self.noise_scheduler.add_noise(latents, noise, + timesteps) + + input_ids = self.tokenize_caption(text).to(self.device) + + # Get the text embedding for conditioning + with torch.no_grad(): + encoder_hidden_states = self.text_encoder(input_ids)[0] + + # Get the target for loss depending on the prediction type + if self.noise_scheduler.config.prediction_type == 'epsilon': + target = noise + elif self.noise_scheduler.config.prediction_type == 'v_prediction': + target = self.noise_scheduler.get_velocity(latents, noise, + timesteps) + else: + raise ValueError( + f'Unknown prediction type {self.noise_scheduler.config.prediction_type}' + ) + + # Predict the noise residual and compute loss + model_pred = self.unet(noisy_latents, timesteps, + encoder_hidden_states).sample + + loss = F.mse_loss(model_pred.float(), target.float(), reduction='mean') + + output = {OutputKeys.LOSS: loss} + return output + + def save_pretrained(self, + target_folder: Union[str, os.PathLike], + save_checkpoint_names: Union[str, List[str]] = None, + save_function: Callable = partial( + save_checkpoint, with_meta=False), + config: Optional[dict] = None, + save_config_function: Callable = save_configuration, + **kwargs): + # Save only the lora model, skip saving and copying the original weights + if self.lora_tune: + pass + else: + super().save_pretrained(target_folder, save_checkpoint_names, + save_function, config, + save_config_function, **kwargs) diff --git a/modelscope/msdatasets/ms_dataset.py b/modelscope/msdatasets/ms_dataset.py index dea65d56..b9ff9971 100644 --- a/modelscope/msdatasets/ms_dataset.py +++ b/modelscope/msdatasets/ms_dataset.py @@ -655,7 +655,7 @@ class MsDataset: def type_converter(self, x): if self.to_tensor: - return torch.tensor(x) + return torch.as_tensor(x) else: return x diff --git a/modelscope/pipelines/multi_modal/__init__.py b/modelscope/pipelines/multi_modal/__init__.py index b28e9a71..11c28fdd 100644 --- a/modelscope/pipelines/multi_modal/__init__.py +++ b/modelscope/pipelines/multi_modal/__init__.py @@ -18,7 +18,7 @@ if TYPE_CHECKING: from .document_vl_embedding_pipeline import DocumentVLEmbeddingPipeline from .video_captioning_pipeline import VideoCaptioningPipeline from .video_question_answering_pipeline import VideoQuestionAnsweringPipeline - from .diffusers_wrapped import StableDiffusionWrapperPipeline, ChineseStableDiffusionPipeline + from .diffusers_wrapped import StableDiffusionPipeline, ChineseStableDiffusionPipeline from .soonet_video_temporal_grounding_pipeline import SOONetVideoTemporalGroundingPipeline from .text_to_video_synthesis_pipeline import TextToVideoSynthesisPipeline from .multimodal_dialogue_pipeline import MultimodalDialoguePipeline @@ -42,7 +42,7 @@ else: 'video_question_answering_pipeline': ['VideoQuestionAnsweringPipeline'], 'diffusers_wrapped': - ['StableDiffusionWrapperPipeline', 'ChineseStableDiffusionPipeline'], + ['StableDiffusionPipeline', 'ChineseStableDiffusionPipeline'], 'soonet_video_temporal_grounding_pipeline': ['SOONetVideoTemporalGroundingPipeline'], 'text_to_video_synthesis_pipeline': ['TextToVideoSynthesisPipeline'], diff --git a/modelscope/pipelines/multi_modal/diffusers_wrapped/__init__.py b/modelscope/pipelines/multi_modal/diffusers_wrapped/__init__.py index 0c9fc5e8..8c5a2632 100644 --- a/modelscope/pipelines/multi_modal/diffusers_wrapped/__init__.py +++ b/modelscope/pipelines/multi_modal/diffusers_wrapped/__init__.py @@ -4,12 +4,12 @@ from typing import TYPE_CHECKING from modelscope.utils.import_utils import LazyImportModule if TYPE_CHECKING: - from .stable_diffusion import StableDiffusionWrapperPipeline + from .stable_diffusion import StableDiffusionPipeline from .stable_diffusion import ChineseStableDiffusionPipeline else: _import_structure = { 'stable_diffusion': - ['StableDiffusionWrapperPipeline', 'ChineseStableDiffusionPipeline'] + ['StableDiffusionPipeline', 'ChineseStableDiffusionPipeline'] } import sys diff --git a/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/__init__.py b/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/__init__.py index 6892877a..5f5d44f3 100644 --- a/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/__init__.py +++ b/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/__init__.py @@ -4,11 +4,11 @@ from typing import TYPE_CHECKING from modelscope.utils.import_utils import LazyImportModule if TYPE_CHECKING: - from .stable_diffusion_pipeline import StableDiffusionWrapperPipeline + from .stable_diffusion_pipeline import StableDiffusionPipeline from .chinese_stable_diffusion_pipeline import ChineseStableDiffusionPipeline else: _import_structure = { - 'stable_diffusion_pipeline': ['StableDiffusionWrapperPipeline'], + 'stable_diffusion_pipeline': ['StableDiffusionPipeline'], 'chinese_stable_diffusion_pipeline': ['ChineseStableDiffusionPipeline'] } diff --git a/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py b/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py index 49b4ef37..c4f8bda8 100644 --- a/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py +++ b/modelscope/pipelines/multi_modal/diffusers_wrapped/stable_diffusion/stable_diffusion_pipeline.py @@ -1,44 +1,49 @@ # Copyright © Alibaba, Inc. and its affiliates. - -from typing import Any, Dict +import os +from typing import Any, Dict, Optional import cv2 import numpy as np import torch -from diffusers import StableDiffusionPipeline +import torchvision.transforms as transforms +from diffusers import \ + StableDiffusionPipeline as DiffuserStableDiffusionPipeline from PIL import Image from modelscope.metainfo import Pipelines +from modelscope.models import Model from modelscope.outputs import OutputKeys +from modelscope.pipelines.base import Pipeline from modelscope.pipelines.builder import PIPELINES from modelscope.pipelines.multi_modal.diffusers_wrapped.diffusers_pipeline import \ DiffusersPipeline from modelscope.utils.constant import Tasks -# Wrap around the diffusers stable diffusion pipeline implementation -# for a unified ModelScope pipeline experience. Native stable diffusion -# pipelines will be implemented in later releases. @PIPELINES.register_module( Tasks.text_to_image_synthesis, module_name=Pipelines.diffusers_stable_diffusion) -class StableDiffusionWrapperPipeline(DiffusersPipeline): +class StableDiffusionPipeline(DiffusersPipeline): - def __init__(self, model: str, device: str = 'gpu', **kwargs): + def __init__(self, model: str, lora_dir: str = None, **kwargs): """ use `model` to create a stable diffusion pipeline Args: - model: model id on modelscope hub. - device: str = 'gpu' + model: model id on modelscope hub or local model dir. """ - super().__init__(model, device, **kwargs) - torch_dtype = kwargs.get('torch_dtype', torch.float32) + self.device = 'cuda' if torch.cuda.is_available() else 'cpu' + # load pipeline + self.pipeline = DiffuserStableDiffusionPipeline.from_pretrained( + model, torch_dtype=torch.float16) + self.pipeline = self.pipeline.to(self.device) + # load lora moudle to unet + if lora_dir is not None: + assert os.path.exists(lora_dir), f"{lora_dir} isn't exist" + self.pipeline.unet.load_attn_procs(lora_dir) - # build upon the diffuser stable diffusion pipeline - self.pipeline = StableDiffusionPipeline.from_pretrained( - model, torch_dtype=torch_dtype) - self.pipeline.to(self.device) + def preprocess(self, inputs: Dict[str, Any], **kwargs) -> Dict[str, Any]: + return inputs def forward(self, inputs: Dict[str, Any], **forward_params) -> Dict[str, Any]: @@ -46,24 +51,14 @@ class StableDiffusionWrapperPipeline(DiffusersPipeline): raise ValueError( f'Expected the input to be a dictionary, but got {type(input)}' ) + if 'text' not in inputs: raise ValueError('input should contain "text", but not found') - return self.pipeline( - prompt=inputs.get('text'), - height=inputs.get('height'), - width=inputs.get('width'), - num_inference_steps=inputs.get('num_inference_steps', 50), - guidance_scale=inputs.get('guidance_scale', 7.5), - negative_prompt=inputs.get('negative_prompt'), - num_images_per_prompt=inputs.get('num_images_per_prompt', 1), - eta=inputs.get('eta', 0.0), - generator=inputs.get('generator'), - latents=inputs.get('latents'), - output_type=inputs.get('output_type', 'pil'), - return_dict=inputs.get('return_dict', True), - callback=inputs.get('callback'), - callback_steps=inputs.get('callback_steps', 1)) + images = self.pipeline( + inputs['text'], num_inference_steps=30, guidance_scale=7.5) + + return images def postprocess(self, inputs: Dict[str, Any], **kwargs) -> Dict[str, Any]: images = [] diff --git a/modelscope/preprocessors/multi_modal.py b/modelscope/preprocessors/multi_modal.py index faf796f4..82d44da8 100644 --- a/modelscope/preprocessors/multi_modal.py +++ b/modelscope/preprocessors/multi_modal.py @@ -48,13 +48,14 @@ class DiffusionImageGenerationPreprocessor(Preprocessor): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.preprocessor_resolution = kwargs.pop('resolution', 512) - self.preprocessor_mean = kwargs.pop('mean', [0.5, 0.5, 0.5]) - self.preprocessor_std = kwargs.pop('std', [0.5, 0.5, 0.5]) + self.preprocessor_mean = kwargs.pop('mean', [0.5]) + self.preprocessor_std = kwargs.pop('std', [0.5]) self.preprocessor_image_keys = set(kwargs.pop('image_keys', [])) self.transform_input = transforms.Compose([ transforms.Resize( self.preprocessor_resolution, interpolation=transforms.InterpolationMode.BILINEAR), + transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(self.preprocessor_mean, self.preprocessor_std), diff --git a/modelscope/trainers/multi_modal/lora_diffusion/__init__.py b/modelscope/trainers/multi_modal/lora_diffusion/__init__.py new file mode 100644 index 00000000..311d2789 --- /dev/null +++ b/modelscope/trainers/multi_modal/lora_diffusion/__init__.py @@ -0,0 +1 @@ +from .lora_diffusion_trainer import LoraDiffusionTrainer diff --git a/modelscope/trainers/multi_modal/lora_diffusion/lora_diffusion_trainer.py b/modelscope/trainers/multi_modal/lora_diffusion/lora_diffusion_trainer.py new file mode 100644 index 00000000..40da164e --- /dev/null +++ b/modelscope/trainers/multi_modal/lora_diffusion/lora_diffusion_trainer.py @@ -0,0 +1,78 @@ +# Copyright 2022-2023 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import Union + +import torch +from diffusers.loaders import AttnProcsLayers +from diffusers.models.attention_processor import LoRAAttnProcessor + +from modelscope.metainfo import Trainers +from modelscope.trainers.builder import TRAINERS +from modelscope.trainers.hooks.checkpoint.checkpoint_hook import CheckpointHook +from modelscope.trainers.hooks.checkpoint.checkpoint_processor import \ + CheckpointProcessor +from modelscope.trainers.optimizer.builder import build_optimizer +from modelscope.trainers.trainer import EpochBasedTrainer +from modelscope.utils.config import ConfigDict + + +class LoraDiffusionCheckpointProcessor(CheckpointProcessor): + + def save_checkpoints(self, + trainer, + checkpoint_path_prefix, + output_dir, + meta=None): + """Save the state dict for lora tune model. + """ + trainer.model.unet = trainer.model.unet.to(torch.float32) + trainer.model.unet.save_attn_procs(output_dir) + + +@TRAINERS.register_module(module_name=Trainers.lora_diffusion) +class LoraDiffusionTrainer(EpochBasedTrainer): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # set lora save checkpoint processor + ckpt_hook = list( + filter(lambda hook: isinstance(hook, CheckpointHook), + self.hooks))[0] + ckpt_hook.set_processor(LoraDiffusionCheckpointProcessor()) + # Set correct lora layers + lora_attn_procs = {} + for name in self.model.unet.attn_processors.keys(): + cross_attention_dim = None if name.endswith( + 'attn1.processor' + ) else self.model.unet.config.cross_attention_dim + if name.startswith('mid_block'): + hidden_size = self.model.unet.config.block_out_channels[-1] + elif name.startswith('up_blocks'): + block_id = int(name[len('up_blocks.')]) + hidden_size = list( + reversed( + self.model.unet.config.block_out_channels))[block_id] + elif name.startswith('down_blocks'): + block_id = int(name[len('down_blocks.')]) + hidden_size = self.model.unet.config.block_out_channels[ + block_id] + + lora_attn_procs[name] = LoRAAttnProcessor( + hidden_size=hidden_size, + cross_attention_dim=cross_attention_dim) + + self.model.unet.set_attn_processor(lora_attn_procs) + + self.lora_layers = AttnProcsLayers(self.model.unet.attn_processors) + + def build_optimizer(self, cfg: ConfigDict, default_args: dict = None): + try: + return build_optimizer( + self.lora_layers.parameters(), + cfg=cfg, + default_args=default_args) + except KeyError as e: + self.logger.error( + f'Build optimizer error, the optimizer {cfg} is a torch native component, ' + f'please check if your torch with version: {torch.__version__} matches the config.' + ) + raise e diff --git a/modelscope/trainers/multi_modal/stable_diffusion/__init__.py b/modelscope/trainers/multi_modal/stable_diffusion/__init__.py new file mode 100644 index 00000000..a23f2880 --- /dev/null +++ b/modelscope/trainers/multi_modal/stable_diffusion/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from .stable_diffusion_trainer import StableDiffusionTrainer diff --git a/modelscope/trainers/multi_modal/stable_diffusion/stable_diffusion_trainer.py b/modelscope/trainers/multi_modal/stable_diffusion/stable_diffusion_trainer.py new file mode 100644 index 00000000..68d7c689 --- /dev/null +++ b/modelscope/trainers/multi_modal/stable_diffusion/stable_diffusion_trainer.py @@ -0,0 +1,30 @@ +# Copyright 2022-2023 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import Union + +import torch +from torch import nn + +from modelscope.metainfo import Trainers +from modelscope.models.base import Model, TorchModel +from modelscope.trainers.builder import TRAINERS +from modelscope.trainers.optimizer.builder import build_optimizer +from modelscope.trainers.trainer import EpochBasedTrainer +from modelscope.utils.config import ConfigDict + + +@TRAINERS.register_module(module_name=Trainers.stable_diffusion) +class StableDiffusionTrainer(EpochBasedTrainer): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def build_optimizer(self, cfg: ConfigDict, default_args: dict = None): + try: + return build_optimizer( + self.model.unet, cfg=cfg, default_args=default_args) + except KeyError as e: + self.logger.error( + f'Build optimizer error, the optimizer {cfg} is a torch native component, ' + f'please check if your torch with version: {torch.__version__} matches the config.' + ) + raise e diff --git a/modelscope/tuners/control_sd_lora.py b/modelscope/tuners/control_sd_lora.py index 2585daa1..a2c53e24 100644 --- a/modelscope/tuners/control_sd_lora.py +++ b/modelscope/tuners/control_sd_lora.py @@ -12,8 +12,8 @@ import torch.nn.functional as F from diffusers.configuration_utils import ConfigMixin, register_to_config from diffusers.models.cross_attention import CrossAttention, LoRALinearLayer from diffusers.models.modeling_utils import ModelMixin -from diffusers.models.resnet import (Downsample2D, Mish, Upsample2D, - downsample_2d, partial, upsample_2d) +from diffusers.models.resnet import (Downsample2D, Upsample2D, downsample_2d, + partial, upsample_2d) from diffusers.models.unet_2d_blocks import \ get_down_block as get_down_block_default from diffusers.utils.outputs import BaseOutput @@ -477,8 +477,10 @@ class ControlLoRACrossAttnProcessor(LoRACrossAttnProcessor): assert self.control_states is not None batch_size, sequence_length, _ = hidden_states.shape - attention_mask = attn.prepare_attention_mask(attention_mask, - sequence_length) + attention_mask = attn.prepare_attention_mask( + attention_mask=attention_mask, + target_length=sequence_length, + batch_size=batch_size) query = attn.to_q(hidden_states) for pre_lora in self.pre_loras: lora_in = query if pre_lora.post_add else hidden_states @@ -627,8 +629,10 @@ class ControlLoRACrossAttnProcessorV2(LoRACrossAttnProcessor): assert self.control_states is not None batch_size, sequence_length, _ = hidden_states.shape - attention_mask = attn.prepare_attention_mask(attention_mask, - sequence_length) + attention_mask = attn.prepare_attention_mask( + attention_mask=attention_mask, + target_length=sequence_length, + batch_size=batch_size) for pre_lora in self.pre_loras: if isinstance(pre_lora, ControlLoRACrossAttnProcessorV2): hidden_states = hidden_states + pre_lora.process_control_states( @@ -783,7 +787,7 @@ class ConvBlock2D(nn.Module): if non_linearity == 'swish': self.nonlinearity = lambda x: F.silu(x) elif non_linearity == 'mish': - self.nonlinearity = Mish() + self.nonlinearity = nn.Mish() elif non_linearity == 'silu': self.nonlinearity = nn.SiLU() diff --git a/modelscope/tuners/sd_lora.py b/modelscope/tuners/sd_lora.py index d740f7c0..feff05f4 100644 --- a/modelscope/tuners/sd_lora.py +++ b/modelscope/tuners/sd_lora.py @@ -90,9 +90,10 @@ class LoRACrossAttnProcessor(nn.Module): attention_mask=None, scale=1.0): batch_size, sequence_length, _ = hidden_states.shape - attention_mask = attn.prepare_attention_mask(attention_mask, - sequence_length, - batch_size) + attention_mask = attn.prepare_attention_mask( + attention_mask=attention_mask, + target_length=sequence_length, + batch_size=batch_size) query = attn.to_q(hidden_states) query = query + scale * self.to_q_lora( diff --git a/modelscope/version.py b/modelscope/version.py index 02464c29..32823e7b 100644 --- a/modelscope/version.py +++ b/modelscope/version.py @@ -1,5 +1,5 @@ # Make sure to modify __release_datetime__ to release time when making official release. -__version__ = '1.6.0' +__version__ = '1.6.2' # default release datetime for branches under active development is set # to be a time far-far-away-into-the-future __release_datetime__ = '2099-10-13 08:56:12' diff --git a/requirements/multi-modal.txt b/requirements/multi-modal.txt index 9d2c3448..fe2d45a1 100644 --- a/requirements/multi-modal.txt +++ b/requirements/multi-modal.txt @@ -1,10 +1,14 @@ accelerate -diffusers>=0.13.1,<0.15.0 +cloudpickle +decord>=0.6.0 +diffusers==0.15.0 +fairseq ftfy>=6.0.3 librosa==0.9.2 opencv-python pycocoevalcap>=1.2 pycocotools>=2.0.4 +pydot # compatible with taming-transformers-rom1504 pytorch_lightning<=1.7.7 rapidfuzz diff --git a/tests/pipelines/test_diffusers_stable_diffusion.py b/tests/pipelines/test_diffusers_stable_diffusion.py index eef677fc..57eae4a3 100644 --- a/tests/pipelines/test_diffusers_stable_diffusion.py +++ b/tests/pipelines/test_diffusers_stable_diffusion.py @@ -17,11 +17,11 @@ class DiffusersStableDiffusionTest(unittest.TestCase): test_input = 'a photo of an astronaut riding a horse on mars' - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_run(self): diffusers_pipeline = pipeline(task=self.task, model=self.model_id) output = diffusers_pipeline({ - 'text': self.test_input, + 'prompt': self.test_input, 'height': 512, 'width': 512 }) diff --git a/tests/trainers/test_efficient_diffusion_tuning_trainer.py b/tests/trainers/test_efficient_diffusion_tuning_trainer.py index 2484e24d..a19bf21d 100644 --- a/tests/trainers/test_efficient_diffusion_tuning_trainer.py +++ b/tests/trainers/test_efficient_diffusion_tuning_trainer.py @@ -39,7 +39,7 @@ class TestEfficientDiffusionTuningTrainer(unittest.TestCase): shutil.rmtree(self.tmp_dir) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_efficient_diffusion_tuning_lora_train(self): model_id = 'damo/multi-modal_efficient-diffusion-tuning-lora' @@ -67,7 +67,7 @@ class TestEfficientDiffusionTuningTrainer(unittest.TestCase): for i in range(self.max_epochs): self.assertIn(f'epoch_{i+1}.pth', results_files) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_efficient_diffusion_tuning_lora_eval(self): model_id = 'damo/multi-modal_efficient-diffusion-tuning-lora' @@ -87,7 +87,7 @@ class TestEfficientDiffusionTuningTrainer(unittest.TestCase): result = trainer.evaluate() print(f'Efficient-diffusion-tuning-lora eval output: {result}.') - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_efficient_diffusion_tuning_control_lora_train(self): model_id = 'damo/multi-modal_efficient-diffusion-tuning-control-lora' @@ -116,7 +116,7 @@ class TestEfficientDiffusionTuningTrainer(unittest.TestCase): for i in range(self.max_epochs): self.assertIn(f'epoch_{i+1}.pth', results_files) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_efficient_diffusion_tuning_control_lora_eval(self): model_id = 'damo/multi-modal_efficient-diffusion-tuning-control-lora' diff --git a/tests/trainers/test_lora_diffusion_trainer.py b/tests/trainers/test_lora_diffusion_trainer.py new file mode 100644 index 00000000..2ba89665 --- /dev/null +++ b/tests/trainers/test_lora_diffusion_trainer.py @@ -0,0 +1,89 @@ +# Copyright 2022-2023 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import os +import shutil +import tempfile +import unittest + +from modelscope.metainfo import Trainers +from modelscope.msdatasets import MsDataset +from modelscope.trainers import build_trainer +from modelscope.utils.constant import DownloadMode +from modelscope.utils.test_utils import test_level + + +class TestLoraDiffusionTrainer(unittest.TestCase): + + def setUp(self): + print(('Testing %s.%s' % (type(self).__name__, self._testMethodName))) + + self.train_dataset = MsDataset.load( + 'buptwq/lora-stable-diffusion-finetune', + split='train', + download_mode=DownloadMode.FORCE_REDOWNLOAD) + self.eval_dataset = MsDataset.load( + 'buptwq/lora-stable-diffusion-finetune', + split='validation', + download_mode=DownloadMode.FORCE_REDOWNLOAD) + + self.max_epochs = 5 + + self.tmp_dir = tempfile.TemporaryDirectory().name + if not os.path.exists(self.tmp_dir): + os.makedirs(self.tmp_dir) + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + super().tearDown() + + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') + def test_lora_diffusion_train(self): + model_id = 'AI-ModelScope/stable-diffusion-v1-5' + model_revision = 'v1.0.6' + + def cfg_modify_fn(cfg): + cfg.train.max_epochs = self.max_epochs + cfg.train.lr_scheduler = { + 'type': 'LambdaLR', + 'lr_lambda': lambda _: 1, + 'last_epoch': -1 + } + cfg.train.optimizer.lr = 1e-4 + return cfg + + kwargs = dict( + model=model_id, + model_revision=model_revision, + work_dir=self.tmp_dir, + train_dataset=self.train_dataset, + eval_dataset=self.eval_dataset, + cfg_modify_fn=cfg_modify_fn) + + trainer = build_trainer( + name=Trainers.lora_diffusion, default_args=kwargs) + trainer.train() + result = trainer.evaluate() + print(f'Lora-diffusion train output: {result}.') + + results_files = os.listdir(self.tmp_dir) + self.assertIn(f'{trainer.timestamp}.log.json', results_files) + + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') + def test_lora_diffusion_eval(self): + model_id = 'AI-ModelScope/stable-diffusion-v1-5' + model_revision = 'v1.0.6' + + kwargs = dict( + model=model_id, + model_revision=model_revision, + work_dir=self.tmp_dir, + train_dataset=None, + eval_dataset=self.eval_dataset) + + trainer = build_trainer( + name=Trainers.lora_diffusion, default_args=kwargs) + result = trainer.evaluate() + print(f'Lora-diffusion eval output: {result}.') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/trainers/test_stable_diffusion_trainer.py b/tests/trainers/test_stable_diffusion_trainer.py new file mode 100644 index 00000000..c3dbcdee --- /dev/null +++ b/tests/trainers/test_stable_diffusion_trainer.py @@ -0,0 +1,89 @@ +# Copyright 2022-2023 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import os +import shutil +import tempfile +import unittest + +from modelscope.metainfo import Trainers +from modelscope.msdatasets import MsDataset +from modelscope.trainers import build_trainer +from modelscope.utils.constant import DownloadMode +from modelscope.utils.test_utils import test_level + + +class TestStableDiffusionTrainer(unittest.TestCase): + + def setUp(self): + print(('Testing %s.%s' % (type(self).__name__, self._testMethodName))) + + self.train_dataset = MsDataset.load( + 'buptwq/lora-stable-diffusion-finetune', + split='train', + download_mode=DownloadMode.FORCE_REDOWNLOAD) + self.eval_dataset = MsDataset.load( + 'buptwq/lora-stable-diffusion-finetune', + split='validation', + download_mode=DownloadMode.FORCE_REDOWNLOAD) + + self.max_epochs = 5 + + self.tmp_dir = tempfile.TemporaryDirectory().name + if not os.path.exists(self.tmp_dir): + os.makedirs(self.tmp_dir) + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + super().tearDown() + + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') + def test_stable_diffusion_train(self): + model_id = 'AI-ModelScope/stable-diffusion-v1-5' + model_revision = 'v1.0.7' + + def cfg_modify_fn(cfg): + cfg.train.max_epochs = self.max_epochs + cfg.train.lr_scheduler = { + 'type': 'LambdaLR', + 'lr_lambda': lambda _: 1, + 'last_epoch': -1 + } + cfg.train.optimizer.lr = 1e-4 + return cfg + + kwargs = dict( + model=model_id, + model_revision=model_revision, + work_dir=self.tmp_dir, + train_dataset=self.train_dataset, + eval_dataset=self.eval_dataset, + cfg_modify_fn=cfg_modify_fn) + + trainer = build_trainer( + name=Trainers.stable_diffusion, default_args=kwargs) + trainer.train() + result = trainer.evaluate() + print(f'Stable-diffusion train output: {result}.') + + results_files = os.listdir(self.tmp_dir) + self.assertIn(f'{trainer.timestamp}.log.json', results_files) + + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') + def test_stable_diffusion_eval(self): + model_id = 'AI-ModelScope/stable-diffusion-v1-5' + model_revision = 'v1.0.7' + + kwargs = dict( + model=model_id, + model_revision=model_revision, + work_dir=self.tmp_dir, + train_dataset=None, + eval_dataset=self.eval_dataset) + + trainer = build_trainer( + name=Trainers.stable_diffusion, default_args=kwargs) + result = trainer.evaluate() + print(f'Stable-diffusion eval output: {result}.') + + +if __name__ == '__main__': + unittest.main() From 54132e92998ca87c87e55efd359ab0c16e411377 Mon Sep 17 00:00:00 2001 From: "wenmeng.zwm" Date: Wed, 21 Jun 2023 16:35:10 +0800 Subject: [PATCH 75/76] update data/test to newest version --- data/test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/test b/data/test index 8d062525..3b92adfa 160000 --- a/data/test +++ b/data/test @@ -1 +1 @@ -Subproject commit 8d0625256b88bdf41655563049a4a68ec1025638 +Subproject commit 3b92adfa9dd558bec74d616a8d3e3583ca95e29f From 3724697b655b80b67a47a907735d5d6c1cf1c4c5 Mon Sep 17 00:00:00 2001 From: "wenmeng.zwm" Date: Wed, 21 Jun 2023 17:30:19 +0800 Subject: [PATCH 76/76] update test data --- data/test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/test b/data/test index 3b92adfa..acc59489 160000 --- a/data/test +++ b/data/test @@ -1 +1 @@ -Subproject commit 3b92adfa9dd558bec74d616a8d3e3583ca95e29f +Subproject commit acc59489d3954fc09cd99d8d4aed818a8d39b283