add more cli tools

提供独立repo库快速接入modelscope的小工具，模板创建，模型上传，一键导入、版本管理等命令

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/11757517

modelscope/cli/cli.py

@@ -3,6 +3,8 @@
 import argparse
 
 from modelscope.cli.download import DownloadCMD
+from modelscope.cli.modelcard import ModelCardCMD
+from modelscope.cli.pipeline import PipelineCMD
 
 
 def run_cmd():
@@ -11,6 +13,8 @@ def run_cmd():
     subparsers = parser.add_subparsers(help='modelscope commands helpers')
 
     DownloadCMD.define_args(subparsers)
+    PipelineCMD.define_args(subparsers)
+    ModelCardCMD.define_args(subparsers)
 
     args = parser.parse_args()
 

modelscope/cli/modelcard.py

@@ -0,0 +1,178 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import shutil
+import tempfile
+from argparse import ArgumentParser
+from string import Template
+
+from modelscope.cli.base import CLICommand
+from modelscope.hub.api import HubApi
+from modelscope.hub.snapshot_download import snapshot_download
+from modelscope.hub.utils.utils import get_endpoint
+from modelscope.utils.logger import get_logger
+
+logger = get_logger()
+
+curren_path = os.path.dirname(os.path.abspath(__file__))
+template_path = os.path.join(curren_path, 'template')
+
+
+def subparser_func(args):
+    """ Fuction which will be called for a specific sub parser.
+    """
+    return ModelCardCMD(args)
+
+
+class ModelCardCMD(CLICommand):
+    name = 'modelcard'
+
+    def __init__(self, args):
+        self.args = args
+        self.api = HubApi()
+        self.api.login(args.access_token)
+        self.model_id = os.path.join(
+            self.args.group_id, self.args.model_id
+        ) if '/' not in self.args.model_id else self.args.model_id
+        self.url = os.path.join(get_endpoint(), self.model_id)
+
+    @staticmethod
+    def define_args(parsers: ArgumentParser):
+        """ define args for create or upload modelcard command.
+        """
+        parser = parsers.add_parser(ModelCardCMD.name)
+        parser.add_argument(
+            '-tk',
+            '--access_token',
+            type=str,
+            required=True,
+            help='the certification of visit ModelScope')
+        parser.add_argument(
+            '-act',
+            '--action',
+            type=str,
+            required=True,
+            choices=['create', 'upload', 'download'],
+            help='the action of api ModelScope[create, upload]')
+        parser.add_argument(
+            '-gid',
+            '--group_id',
+            type=str,
+            default='damo',
+            help='the group name of ModelScope, eg, damo')
+        parser.add_argument(
+            '-mid',
+            '--model_id',
+            type=str,
+            required=True,
+            help='the model name of ModelScope')
+        parser.add_argument(
+            '-vis',
+            '--visibility',
+            type=int,
+            default=5,
+            help='the visibility of ModelScope')
+        parser.add_argument(
+            '-lic',
+            '--license',
+            type=str,
+            default='Apache License 2.0',
+            help='the license of visit ModelScope')
+        parser.add_argument(
+            '-ch',
+            '--chinese_name',
+            type=str,
+            default='这是我的第一个模型',
+            help='the chinese name of ModelScope')
+        parser.add_argument(
+            '-md',
+            '--model_dir',
+            type=str,
+            default='.',
+            help='the model_dir of configuration.json')
+        parser.add_argument(
+            '-vt',
+            '--version_tag',
+            type=str,
+            default=None,
+            help='the tag of uploaded model')
+        parser.add_argument(
+            '-vi',
+            '--version_info',
+            type=str,
+            default=None,
+            help='the info of uploaded model')
+        parser.set_defaults(func=subparser_func)
+
+    def create_model(self):
+        from modelscope.hub.constants import Licenses, ModelVisibility
+        visibilities = [
+            getattr(ModelVisibility, attr) for attr in dir(ModelVisibility)
+            if not attr.startswith('__')
+        ]
+        if self.args.visibility not in visibilities:
+            raise ValueError('The access_token must in %s!' % visibilities)
+        licenses = [
+            getattr(Licenses, attr) for attr in dir(Licenses)
+            if not attr.startswith('__')
+        ]
+        if self.args.license not in licenses:
+            raise ValueError('The license must in %s!' % licenses)
+        try:
+            self.api.get_model(self.model_id)
+        except Exception as e:
+            logger.info('>>> %s' % type(e))
+            self.api.create_model(
+                model_id=self.model_id,
+                visibility=self.args.visibility,
+                license=self.args.license,
+                chinese_name=self.args.chinese_name,
+            )
+        self.pprint()
+
+    def get_model_url(self):
+        return self.api.get_model_url(self.model_id)
+
+    def push_model(self, tpl_dir='readme.tpl'):
+        from modelscope.hub.repository import Repository
+        if self.args.version_tag and self.args.version_info:
+            clone_dir = tempfile.TemporaryDirectory().name
+            repo = Repository(clone_dir, clone_from=self.model_id)
+            repo.tag_and_push(self.args.version_tag, self.args.version_info)
+            shutil.rmtree(clone_dir)
+        else:
+            cfg_file = os.path.join(self.args.model_dir, 'README.md')
+            if not os.path.exists(cfg_file):
+                with open(os.path.join(template_path,
+                                       tpl_dir)) as tpl_file_path:
+                    tpl = Template(tpl_file_path.read())
+                    f = open(cfg_file, 'w')
+                    f.write(tpl.substitute(model_id=self.model_id))
+                    f.close()
+            self.api.push_model(
+                model_id=self.model_id,
+                model_dir=self.args.model_dir,
+                visibility=self.args.visibility,
+                license=self.args.license,
+                chinese_name=self.args.chinese_name)
+        self.pprint()
+
+    def pprint(self):
+        logger.info('>>> Clone the model_git < %s >, commit and push it.'
+                    % self.get_model_url())
+        logger.info('>>> Open the url < %s >, check and read it.' % self.url)
+        logger.info('>>> Visit the model_id < %s >, download and run it.'
+                    % self.model_id)
+
+    def execute(self):
+        if self.args.action == 'create':
+            self.create_model()
+        elif self.args.action == 'upload':
+            self.push_model()
+        elif self.args.action == 'download':
+            snapshot_download(
+                self.model_id,
+                cache_dir=self.args.model_dir,
+                revision=self.args.version_tag)
+        else:
+            raise ValueError(
+                'The parameter of action must be in [create, upload]')

modelscope/cli/pipeline.py

@@ -0,0 +1,127 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+from argparse import ArgumentParser
+from string import Template
+
+from modelscope.cli.base import CLICommand
+from modelscope.utils.logger import get_logger
+
+logger = get_logger()
+
+curren_path = os.path.dirname(os.path.abspath(__file__))
+template_path = os.path.join(curren_path, 'template')
+
+
+def subparser_func(args):
+    """ Fuction which will be called for a specific sub parser.
+    """
+    return PipelineCMD(args)
+
+
+class PipelineCMD(CLICommand):
+    name = 'pipeline'
+
+    def __init__(self, args):
+        self.args = args
+
+    @staticmethod
+    def define_args(parsers: ArgumentParser):
+        """ define args for create pipeline template command.
+        """
+        parser = parsers.add_parser(PipelineCMD.name)
+        parser.add_argument(
+            '-act',
+            '--action',
+            type=str,
+            required=True,
+            choices=['create'],
+            help='the action of command  pipeline[create]')
+        parser.add_argument(
+            '-tpl',
+            '--tpl_file_path',
+            type=str,
+            default='template.tpl',
+            help='the template be selected for ModelScope[template.tpl]')
+        parser.add_argument(
+            '-s',
+            '--save_file_path',
+            type=str,
+            default='./',
+            help='the name of custom template be saved for ModelScope')
+        parser.add_argument(
+            '-f',
+            '--filename',
+            type=str,
+            default='ms_wrapper.py',
+            help='the init name of custom template be saved for ModelScope')
+        parser.add_argument(
+            '-t',
+            '--task_name',
+            type=str,
+            required=True,
+            help='the unique task_name for ModelScope')
+        parser.add_argument(
+            '-m',
+            '--model_name',
+            type=str,
+            default='MyCustomModel',
+            help='the class of model name for ModelScope')
+        parser.add_argument(
+            '-p',
+            '--preprocessor_name',
+            type=str,
+            default='MyCustomPreprocessor',
+            help='the class of preprocessor name for ModelScope')
+        parser.add_argument(
+            '-pp',
+            '--pipeline_name',
+            type=str,
+            default='MyCustomPipeline',
+            help='the class of pipeline name for ModelScope')
+        parser.add_argument(
+            '-config',
+            '--configuration_path',
+            type=str,
+            default='./',
+            help='the path of configuration.json for ModelScope')
+        parser.set_defaults(func=subparser_func)
+
+    def create_template(self):
+        if self.args.tpl_file_path not in os.listdir(template_path):
+            tpl_file_path = self.args.tpl_file_path
+        else:
+            tpl_file_path = os.path.join(template_path,
+                                         self.args.tpl_file_path)
+        if not os.path.exists(tpl_file_path):
+            raise ValueError('%s not exists!' % tpl_file_path)
+
+        save_file_path = self.args.save_file_path if self.args.save_file_path != './' else os.getcwd(
+        )
+        os.makedirs(save_file_path, exist_ok=True)
+        if not self.args.filename.endswith('.py'):
+            raise ValueError('the FILENAME must end with .py ')
+        save_file_name = self.args.filename
+        save_pkl_path = os.path.join(save_file_path, save_file_name)
+
+        if not self.args.configuration_path.endswith('/'):
+            self.args.configuration_path = self.args.configuration_path + '/'
+
+        lines = []
+        with open(tpl_file_path) as tpl_file:
+            tpl = Template(tpl_file.read())
+            lines.append(tpl.substitute(**vars(self.args)))
+
+        with open(save_pkl_path, 'w') as save_file:
+            save_file.writelines(lines)
+
+        logger.info('>>> Configuration be saved in %s/%s' %
+                    (self.args.configuration_path, 'configuration.json'))
+        logger.info('>>> Task_name: %s, Created in %s' %
+                    (self.args.task_name, save_pkl_path))
+        logger.info('Open the file < %s >, update and run it.' % save_pkl_path)
+
+    def execute(self):
+        if self.args.action == 'create':
+            self.create_template()
+        else:
+            raise ValueError('The parameter of action must be in [create]')

modelscope/cli/template/readme.tpl

@@ -0,0 +1,10 @@
+---
+license: Apache License 2.0
+---
+###### 该模型当前使用的是默认介绍模版，处于“预发布”阶段，页面仅限所有者可见。
+###### 请根据[模型贡献文档说明](https://www.modelscope.cn/docs/%E5%A6%82%E4%BD%95%E6%92%B0%E5%86%99%E5%A5%BD%E7%94%A8%E7%9A%84%E6%A8%A1%E5%9E%8B%E5%8D%A1%E7%89%87)，及时完善模型卡片内容。ModelScope平台将在模型卡片完善后展示。谢谢您的理解。
+
+#### Clone with HTTP
+```bash
+ git clone https://www.modelscope.cn/${model_id}.git
+```

modelscope/cli/template/template.tpl

@@ -0,0 +1,139 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import torch
+
+from modelscope.models.base import TorchModel
+from modelscope.preprocessors.base import Preprocessor
+from modelscope.pipelines.base import Model, Pipeline
+from modelscope.utils.config import Config
+from modelscope.pipelines.builder import PIPELINES
+from modelscope.preprocessors.builder import PREPROCESSORS
+from modelscope.models.builder import MODELS
+
+
+@MODELS.register_module('${task_name}', module_name='my-custom-model')
+class ${model_name}(TorchModel):
+
+    def __init__(self, model_dir, *args, **kwargs):
+        super().__init__(model_dir, *args, **kwargs)
+        self.model = self.init_model(**kwargs)
+
+    def forward(self, input_tensor, **forward_params):
+        return self.model(input_tensor, **forward_params)
+
+    def init_model(self, **kwargs):
+        """Provide default implementation based on TorchModel and user can reimplement it.
+            include init model and load ckpt from the model_dir, maybe include preprocessor
+            if nothing to do, then return lambdx x: x
+        """
+        return lambda x: x
+
+
+@PREPROCESSORS.register_module('${task_name}', module_name='my-custom-preprocessor')
+class ${preprocessor_name}(Preprocessor):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.trainsforms = self.init_preprocessor(**kwargs)
+
+    def __call__(self, results):
+        return self.trainsforms(results)
+
+    def init_preprocessor(self, **kwarg):
+        """ Provide default implementation based on preprocess_cfg and user can reimplement it.
+            if nothing to do, then return lambdx x: x
+        """
+        return lambda x: x
+
+
+@PIPELINES.register_module('${task_name}', module_name='my-custom-pipeline')
+class ${pipeline_name}(Pipeline):
+    """ Give simple introduction to this pipeline.
+
+    Examples:
+
+    >>> from modelscope.pipelines import pipeline
+    >>> input = "Hello, ModelScope!"
+    >>> my_pipeline = pipeline('my-task', 'my-model-id')
+    >>> result = my_pipeline(input)
+
+    """
+
+    def __init__(self, model, preprocessor=None, **kwargs):
+        """
+        use `model` and `preprocessor` to create a custom pipeline for prediction
+        Args:
+            model: model id on modelscope hub.
+            preprocessor: the class of method be init_preprocessor
+        """
+        super().__init__(model=model, auto_collate=False)
+        assert isinstance(model, str) or isinstance(model, Model), \
+            'model must be a single str or Model'
+        if isinstance(model, str):
+            pipe_model = Model.from_pretrained(model)
+        elif isinstance(model, Model):
+            pipe_model = model
+        else:
+            raise NotImplementedError
+        pipe_model.eval()
+
+        if preprocessor is None:
+            preprocessor = ${preprocessor_name}()
+        super().__init__(model=pipe_model, preprocessor=preprocessor, **kwargs)
+
+    def _sanitize_parameters(self, **pipeline_parameters):
+        """
+        this method should sanitize the keyword args to preprocessor params,
+        forward params and postprocess params on '__call__' or '_process_single' method
+        considered to be a normal classmethod with default implementation / output
+
+        Default Returns:
+            Dict[str, str]:  preprocess_params = {}
+            Dict[str, str]:  forward_params = {}
+            Dict[str, str]:  postprocess_params = pipeline_parameters
+        """
+        return {}, pipeline_parameters, {}
+
+    def _check_input(self, inputs):
+        pass
+
+    def _check_output(self, outputs):
+        pass
+
+    def forward(self, inputs, **forward_params):
+        """ Provide default implementation using self.model and user can reimplement it
+        """
+        return super().forward(inputs, **forward_params)
+
+    def postprocess(self, inputs):
+        """ If current pipeline support model reuse, common postprocess
+            code should be write here.
+
+        Args:
+            inputs:  input data
+
+        Return:
+            dict of results:  a dict containing outputs of model, each
+                output should have the standard output name.
+        """
+        return inputs
+
+
+# Tips: usr_config_path is the temporary save configuration location， after upload modelscope hub, it is the model_id
+usr_config_path = '${configuration_path}'
+config = Config({
+    'framework': 'pytorch',
+    'task': '${task_name}',
+    'model': {'type': 'my-custom-model'},
+    "pipeline": {"type": "my-custom-pipeline"}
+})
+config.dump('${configuration_path}' + 'configuration.json')
+
+if __name__ == "__main__":
+    from modelscope.models import Model
+    from modelscope.pipelines import pipeline
+    # model = Model.from_pretrained(usr_config_path)
+    input = "Hello, ModelScope!"
+    inference = pipeline('${task_name}', model=usr_config_path)
+    output = inference(input)
+    print(output)

tests/cli/test_custom_pipeline_cmd.py

@@ -0,0 +1,23 @@
+import os
+import shutil
+import subprocess
+import tempfile
+import unittest
+import uuid
+
+
+class ModelUploadCMDTest(unittest.TestCase):
+
+    def setUp(self):
+        self.task_name = 'task-%s' % (uuid.uuid4().hex)
+        print(self.task_name)
+
+    def test_upload_modelcard(self):
+        cmd = f'python -m modelscope.cli.cli pipeline --action create --task_name {self.task_name} '
+        stat, output = subprocess.getstatusoutput(cmd)
+        if stat != 0:
+            print(output)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/cli/test_modelcard_cmd.py

@@ -0,0 +1,54 @@
+import os
+import os.path as osp
+import shutil
+import subprocess
+import tempfile
+import unittest
+import uuid
+
+from modelscope.hub.api import HubApi
+from modelscope.utils.test_utils import TEST_ACCESS_TOKEN1, TEST_MODEL_ORG
+
+
+class ModelUploadCMDTest(unittest.TestCase):
+
+    def setUp(self):
+        print(('Testing %s.%s' % (type(self).__name__, self._testMethodName)))
+        self.tmp_dir = tempfile.TemporaryDirectory().name
+        if not os.path.exists(self.tmp_dir):
+            os.makedirs(self.tmp_dir)
+        print(self.tmp_dir)
+        self.api = HubApi()
+        self.api.login(TEST_ACCESS_TOKEN1)
+        self.task_name = 'task-%s' % (uuid.uuid4().hex)
+        self.model_name = 'op-%s' % (uuid.uuid4().hex)
+        self.model_id = '%s/%s' % (TEST_MODEL_ORG, self.model_name)
+        print(self.tmp_dir, self.task_name, self.model_name)
+
+    def tearDown(self):
+        self.api.delete_model(model_id=self.model_id)
+        shutil.rmtree(self.tmp_dir)
+        super().tearDown()
+
+    def test_upload_modelcard(self):
+        cmd = f'python -m modelscope.cli.cli pipeline --action create --task_name {self.task_name} ' \
+              f'--save_file_path {self.tmp_dir} --configuration_path {self.tmp_dir}'
+        stat, output = subprocess.getstatusoutput(cmd)
+        if stat != 0:
+            print(output)
+
+        cmd = f'python {self.tmp_dir}/ms_wrapper.py'
+        stat, output = subprocess.getstatusoutput(cmd)
+        if stat != 0:
+            print(output)
+        self.assertEqual(stat, 0)
+
+        cmd = f'python -m modelscope.cli.cli modelcard --action upload -tk {TEST_ACCESS_TOKEN1} ' \
+              f'--model_id {self.model_id} --model_dir {self.tmp_dir}'
+        stat, output = subprocess.getstatusoutput(cmd)
+        if stat != 0:
+            print(output)
+
+
+if __name__ == '__main__':
+    unittest.main()