diff --git a/docs/source/command.md b/docs/source/command.md new file mode 100644 index 00000000..ec44c761 --- /dev/null +++ b/docs/source/command.md @@ -0,0 +1,157 @@ +# ModelScope command line usage +## Supported commands +```bash +modelscope --help +usage: modelscope [] + +positional arguments: + {download,plugin,pipeline,modelcard,model,server,login} + modelscope commands helpers + +options: + -h, --help show this help message and exit + +``` +## login +```bash +modelscope login --help +usage: modelscope [] login [-h] --token TOKEN + +options: + -h, --help show this help message and exit + --token TOKEN The Access Token for modelscope. +``` +Get access token: [我的页面](https://modelscope.cn/my/myaccesstoken)获取**SDK 令牌** + + +## download model +```bash +modelscope download --help + + usage: modelscope [] download [-h] --model MODEL [--revision REVISION] [--cache_dir CACHE_DIR] [--local_dir LOCAL_DIR] [--include [INCLUDE ...]] [--exclude [EXCLUDE ...]] [files ...] + + positional arguments: + files Specify relative path to the repository file(s) to download.(e.g 'tokenizer.json', 'onnx/decoder_model.onnx'). + + options: + -h, --help show this help message and exit + --model MODEL The model id to be downloaded. + --revision REVISION Revision of the model. + --cache_dir CACHE_DIR + Cache directory to save model. + --local_dir LOCAL_DIR + File will be downloaded to local location specified bylocal_dir, in this case, cache_dir parameter will be ignored. + --include [INCLUDE ...] + Glob patterns to match files to download.Ignored if file is specified + --exclude [EXCLUDE ...] + Glob patterns to exclude from files to download.Ignored if file is specified +``` +## Usage Examples + +Command Examples（[gpt2](https://www.modelscope.cn/models/AI-ModelScope/gpt2/files)） + +### Specify downloading of a single file +```bash + modelscope download --model 'AI-ModelScope/gpt2' 64.tflite +``` + +### Specify multiple files to download +```bash + modelscope download --model 'AI-ModelScope/gpt2' 64.tflite config.json +``` +### Specify certain files to download  +```bash + modelscope download --model 'AI-ModelScope/gpt2' --include 'onnx/*' '*.tflite' +``` +### Filter specified files +```bash + modelscope download --model 'AI-ModelScope/gpt2' --exclude 'onnx/*' '*.tflite'  +``` +### Specify the download cache directory +```bash + modelscope download --model 'AI-ModelScope/gpt2' --include '*.json' --cache_dir './cache_dir' +``` +   The model files will be downloaded to cache\_dir/AI-ModelScope/gpt2/ + +### Specify the local directory for downloading     +```bash + modelscope download --model 'AI-ModelScope/gpt2' --include '*.json' --cache_dir './local_dir' +``` +  The model files will be downloaded to ./local\_dir + +If both the local directory and the cache directory are specified, the local directory will take precedence. + +## model operation +Supports creating models and uploading model files. +```bash +modelscope model --help +usage: modelscope [] modelcard [-h] [-tk ACCESS_TOKEN] -act {create,upload,download} [-gid GROUP_ID] -mid MODEL_ID [-vis VISIBILITY] [-lic LICENSE] [-ch CHINESE_NAME] [-md MODEL_DIR] [-vt VERSION_TAG] [-vi VERSION_INFO] + +options: + -h, --help show this help message and exit + -tk ACCESS_TOKEN, --access_token ACCESS_TOKEN + the certification of visit ModelScope + -act {create,upload,download}, --action {create,upload,download} + the action of api ModelScope[create, upload] + -gid GROUP_ID, --group_id GROUP_ID + the group name of ModelScope, eg, damo + -mid MODEL_ID, --model_id MODEL_ID + the model name of ModelScope + -vis VISIBILITY, --visibility VISIBILITY + the visibility of ModelScope[PRIVATE: 1, INTERNAL:3, PUBLIC:5] + -lic LICENSE, --license LICENSE + the license of visit ModelScope[Apache License 2.0|GPL-2.0|GPL-3.0|LGPL-2.1|LGPL-3.0|AFL-3.0|ECL-2.0|MIT] + -ch CHINESE_NAME, --chinese_name CHINESE_NAME + the chinese name of ModelScope + -md MODEL_DIR, --model_dir MODEL_DIR + the model_dir of configuration.json + -vt VERSION_TAG, --version_tag VERSION_TAG + the tag of uploaded model + -vi VERSION_INFO, --version_info VERSION_INFO + the info of uploaded model +``` + +### Create model +```bash + modelscope model -act create -gid 'YOUR_GROUP_ID' -mid 'THE_MODEL_ID' -vis 1 -lic 'MIT' -ch '中文名字' +``` +Will create model THE_MODEL_ID in www.modelscope.cn + +### Upload model files +```bash + modelscope model -act upload -gid citest -mid 'testt1' -md modelfiles/ -vt 'v0.0.1' -vi 'upload model files' +``` + +## Pipeline +Create the template files needed for pipeline. + +```bash +modelscope pipeline --help +usage: modelscope [] pipeline [-h] -act {create} [-tpl TPL_FILE_PATH] [-s SAVE_FILE_PATH] [-f FILENAME] -t TASK_NAME [-m MODEL_NAME] [-p PREPROCESSOR_NAME] [-pp PIPELINE_NAME] [-config CONFIGURATION_PATH] + +options: + -h, --help show this help message and exit + -act {create}, --action {create} + the action of command pipeline[create] + -tpl TPL_FILE_PATH, --tpl_file_path TPL_FILE_PATH + the template be selected for ModelScope[template.tpl] + -s SAVE_FILE_PATH, --save_file_path SAVE_FILE_PATH + the name of custom template be saved for ModelScope + -f FILENAME, --filename FILENAME + the init name of custom template be saved for ModelScope + -t TASK_NAME, --task_name TASK_NAME + the unique task_name for ModelScope + -m MODEL_NAME, --model_name MODEL_NAME + the class of model name for ModelScope + -p PREPROCESSOR_NAME, --preprocessor_name PREPROCESSOR_NAME + the class of preprocessor name for ModelScope + -pp PIPELINE_NAME, --pipeline_name PIPELINE_NAME + the class of pipeline name for ModelScope + -config CONFIGURATION_PATH, --configuration_path CONFIGURATION_PATH + the path of configuration.json for ModelScope +``` + +### Create pipeline files +```bash + modelscope pipeline -act 'create' -t 'THE_PIPELINE_TASK' -m 'THE_MODEL_NAME' -pp 'THE_PIPELINE_NAME' +``` diff --git a/modelscope/cli/cli.py b/modelscope/cli/cli.py index 9e690c8c..5c60da8d 100644 --- a/modelscope/cli/cli.py +++ b/modelscope/cli/cli.py @@ -1,6 +1,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import argparse +import logging from modelscope.cli.download import DownloadCMD from modelscope.cli.login import LoginCMD @@ -8,6 +9,9 @@ from modelscope.cli.modelcard import ModelCardCMD from modelscope.cli.pipeline import PipelineCMD from modelscope.cli.plugins import PluginsCMD from modelscope.cli.server import ServerCMD +from modelscope.utils.logger import get_logger + +logger = get_logger(log_level=logging.WARNING) def run_cmd(): diff --git a/modelscope/cli/modelcard.py b/modelscope/cli/modelcard.py index 5e2b6580..646cf1b0 100644 --- a/modelscope/cli/modelcard.py +++ b/modelscope/cli/modelcard.py @@ -1,4 +1,5 @@ # Copyright (c) Alibaba, Inc. and its affiliates. +import logging import os import shutil import tempfile @@ -11,7 +12,7 @@ from modelscope.hub.snapshot_download import snapshot_download from modelscope.hub.utils.utils import get_endpoint from modelscope.utils.logger import get_logger -logger = get_logger() +logger = get_logger(log_level=logging.WARNING) current_path = os.path.dirname(os.path.abspath(__file__)) template_path = os.path.join(current_path, 'template') @@ -29,7 +30,8 @@ class ModelCardCMD(CLICommand): def __init__(self, args): self.args = args self.api = HubApi() - self.api.login(args.access_token) + if args.access_token: + self.api.login(args.access_token) self.model_id = os.path.join( self.args.group_id, self.args.model_id ) if '/' not in self.args.model_id else self.args.model_id @@ -39,12 +41,12 @@ class ModelCardCMD(CLICommand): def define_args(parsers: ArgumentParser): """ define args for create or upload modelcard command. """ - parser = parsers.add_parser(ModelCardCMD.name) + parser = parsers.add_parser(ModelCardCMD.name, aliases=['model']) parser.add_argument( '-tk', '--access_token', type=str, - required=True, + required=False, help='the certification of visit ModelScope') parser.add_argument( '-act', @@ -70,13 +72,15 @@ class ModelCardCMD(CLICommand): '--visibility', type=int, default=5, - help='the visibility of ModelScope') + help= + 'the visibility of ModelScope[PRIVATE: 1, INTERNAL:3, PUBLIC:5]') parser.add_argument( '-lic', '--license', type=str, default='Apache License 2.0', - help='the license of visit ModelScope') + help='the license of visit ModelScope[Apache License 2.0|' + 'GPL-2.0|GPL-3.0|LGPL-2.1|LGPL-3.0|AFL-3.0|ECL-2.0|MIT]') parser.add_argument( '-ch', '--chinese_name', diff --git a/modelscope/cli/pipeline.py b/modelscope/cli/pipeline.py index 793632e0..2b6f7951 100644 --- a/modelscope/cli/pipeline.py +++ b/modelscope/cli/pipeline.py @@ -1,4 +1,5 @@ # Copyright (c) Alibaba, Inc. and its affiliates. +import logging import os from argparse import ArgumentParser from string import Template @@ -6,7 +7,7 @@ from string import Template from modelscope.cli.base import CLICommand from modelscope.utils.logger import get_logger -logger = get_logger() +logger = get_logger(log_level=logging.WARNING) current_path = os.path.dirname(os.path.abspath(__file__)) template_path = os.path.join(current_path, 'template') diff --git a/modelscope/cli/server.py b/modelscope/cli/server.py index 2925d68f..0f8b8132 100644 --- a/modelscope/cli/server.py +++ b/modelscope/cli/server.py @@ -1,4 +1,5 @@ # Copyright (c) Alibaba, Inc. and its affiliates. +import logging import os from argparse import ArgumentParser from string import Template @@ -9,7 +10,7 @@ from modelscope.cli.base import CLICommand from modelscope.server.api_server import add_server_args, get_app from modelscope.utils.logger import get_logger -logger = get_logger() +logger = get_logger(log_level=logging.WARNING) current_path = os.path.dirname(os.path.abspath(__file__)) template_path = os.path.join(current_path, 'template') diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index f235a62d..5ed4ed99 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -1108,7 +1108,7 @@ class ModelScopeConfig: cookies = pickle.load(f) for cookie in cookies: if cookie.is_expired(): - logger.warning( + logger.debug( 'Authentication has expired, ' 'please re-login if you need to access private models or datasets.') return None diff --git a/modelscope/hub/errors.py b/modelscope/hub/errors.py index 6831bd8a..8258399d 100644 --- a/modelscope/hub/errors.py +++ b/modelscope/hub/errors.py @@ -1,5 +1,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. +import logging from http import HTTPStatus import requests @@ -8,7 +9,7 @@ from requests.exceptions import HTTPError from modelscope.hub.constants import MODELSCOPE_REQUEST_ID from modelscope.utils.logger import get_logger -logger = get_logger() +logger = get_logger(log_level=logging.WARNING) class NotSupportError(Exception): diff --git a/modelscope/models/cv/image_to_3d/ldm/modules/encoders/clip/clip.py b/modelscope/models/cv/image_to_3d/ldm/modules/encoders/clip/clip.py index c61c3432..41345249 100644 --- a/modelscope/models/cv/image_to_3d/ldm/modules/encoders/clip/clip.py +++ b/modelscope/models/cv/image_to_3d/ldm/modules/encoders/clip/clip.py @@ -4,9 +4,9 @@ import urllib import warnings from typing import Any, List, Union +import packaging import torch from PIL import Image -from pkg_resources import packaging from torchvision.transforms import (CenterCrop, Compose, Normalize, Resize, ToTensor) from tqdm import tqdm diff --git a/modelscope/models/cv/text_driven_segmentation/clip.py b/modelscope/models/cv/text_driven_segmentation/clip.py index 1cec5f39..9dd40d0e 100644 --- a/modelscope/models/cv/text_driven_segmentation/clip.py +++ b/modelscope/models/cv/text_driven_segmentation/clip.py @@ -8,9 +8,10 @@ import urllib import warnings from typing import Any, List, Union +import packaging +import packaging.version import torch from PIL import Image -from pkg_resources import packaging from torchvision.transforms import (CenterCrop, Compose, Normalize, Resize, ToTensor) from tqdm import tqdm diff --git a/modelscope/models/multi_modal/ofa_for_text_to_image_synthesis_model.py b/modelscope/models/multi_modal/ofa_for_text_to_image_synthesis_model.py index 76ab1170..e6e7d9ac 100644 --- a/modelscope/models/multi_modal/ofa_for_text_to_image_synthesis_model.py +++ b/modelscope/models/multi_modal/ofa_for_text_to_image_synthesis_model.py @@ -6,10 +6,10 @@ from typing import Any, Dict import json import numpy as np +import packaging import torch import torch.cuda from PIL import Image -from pkg_resources import packaging from taming.models.vqgan import GumbelVQ, VQModel from torchvision.transforms import (CenterCrop, Compose, Normalize, Resize, ToTensor) diff --git a/modelscope/server/models/output.py b/modelscope/server/models/output.py index 39abcac2..fa86f49a 100644 --- a/modelscope/server/models/output.py +++ b/modelscope/server/models/output.py @@ -3,12 +3,12 @@ from http import HTTPStatus from typing import Generic, Optional, Type, TypeVar import json -from pydantic.generics import GenericModel +from pydantic import BaseModel ResultType = TypeVar('ResultType') -class ApiResponse(GenericModel, Generic[ResultType]): +class ApiResponse(BaseModel, Generic[ResultType]): Code: Optional[int] = HTTPStatus.OK Success: Optional[bool] = True RequestId: Optional[str] = '' diff --git a/modelscope/utils/ast_utils.py b/modelscope/utils/ast_utils.py index 05e2e237..6b9ba463 100644 --- a/modelscope/utils/ast_utils.py +++ b/modelscope/utils/ast_utils.py @@ -2,6 +2,7 @@ import ast import hashlib +import logging import os import os.path as osp import time @@ -23,7 +24,7 @@ from modelscope.utils.file_utils import get_modelscope_cache_dir from modelscope.utils.logger import get_logger from modelscope.utils.registry import default_group -logger = get_logger() +logger = get_logger(log_level=logging.WARNING) storage = LocalStorage() p = Path(__file__) diff --git a/modelscope/utils/import_utils.py b/modelscope/utils/import_utils.py index 8e707f06..f1a9ad70 100644 --- a/modelscope/utils/import_utils.py +++ b/modelscope/utils/import_utils.py @@ -3,6 +3,7 @@ import ast import functools import importlib +import logging import os import os.path as osp import sys @@ -25,7 +26,7 @@ if sys.version_info < (3, 8): else: import importlib.metadata as importlib_metadata -logger = get_logger() +logger = get_logger(log_level=logging.WARNING) AST_INDEX = None diff --git a/modelscope/utils/input_output.py b/modelscope/utils/input_output.py index 96fe69a8..37e875bc 100644 --- a/modelscope/utils/input_output.py +++ b/modelscope/utils/input_output.py @@ -8,12 +8,9 @@ from io import BytesIO from typing import Any from urllib.parse import urlparse -import cv2 import json import numpy as np -from modelscope.hub.api import HubApi -from modelscope.hub.errors import NotExistError from modelscope.hub.file_download import model_file_download from modelscope.outputs.outputs import (TASK_OUTPUTS, OutputKeys, OutputTypes, OutputTypeSchema) @@ -714,6 +711,7 @@ def service_base64_input_to_pipeline_input(task_name, body): def encode_numpy_image_to_base64(image): + import cv2 _, img_encode = cv2.imencode('.png', image) bytes_data = img_encode.tobytes() base64_str = str(base64.b64encode(bytes_data), 'utf-8') diff --git a/modelscope/utils/logger.py b/modelscope/utils/logger.py index 48f1dbee..be1c24d3 100644 --- a/modelscope/utils/logger.py +++ b/modelscope/utils/logger.py @@ -28,6 +28,8 @@ def get_logger(log_file: Optional[str] = None, logger.propagate = False if logger_name in init_loggers: add_file_handler_if_needed(logger, log_file, file_mode, log_level) + if logger.level != log_level: + logger.setLevel(log_level) return logger # handle duplicate logs to the console