[to #42373878] refactor maaslib to modelscope

1.  refactor maaslib to modelscope
2.  fix UT error
3.  support pipeline which does not register default model

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/8988388
This commit is contained in:
wenmeng.zwm
2022-06-09 20:16:26 +08:00
parent dd00195814
commit 1f6b376599
133 changed files with 804 additions and 573 deletions

View File

@@ -4,5 +4,5 @@ rm -rf build
# update api rst # update api rst
#rm -rf source/api/ #rm -rf source/api/
#sphinx-apidoc --module-first -o source/api/ ../maas_lib/ #sphinx-apidoc --module-first -o source/api/ ../modelscope/
make html make html

View File

@@ -1,3 +1,3 @@
yapf -r -i maas_lib/ configs/ tests/ setup.py yapf -r -i modelscope/ configs/ tests/ setup.py
isort -rc maas_lib/ configs/ tests/ setup.py isort -rc modelscope/ configs/ tests/ setup.py
flake8 maas_lib/ configs/ tests/ setup.py flake8 modelscope/ configs/ tests/ setup.py

View File

@@ -1,4 +1,4 @@
Copyright 2022-2023 Alibaba MaaS. All rights reserved. Copyright 2022-2023 Alibaba ModelScope. All rights reserved.
Apache License Apache License
Version 2.0, January 2004 Version 2.0, January 2004
@@ -188,7 +188,7 @@ Copyright 2022-2023 Alibaba MaaS. All rights reserved.
same "printed page" as the copyright notice for easier same "printed page" as the copyright notice for easier
identification within third-party archives. identification within third-party archives.
Copyright 2020-2022 Alibaba MaaS. Copyright 2020-2022 Alibaba ModelScope.
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.

View File

@@ -1 +1 @@
recursive-include maas_lib/configs *.py recursive-include modelscope/configs *.py

View File

@@ -1,6 +1,6 @@
# Introduction # Introduction
MaaS library is targeted to support training, evaluation and inference for the state of the art models provided by Mind and further support third-party models provided by users outside alibaba. ModelScope library is targeted to support training, evaluation and inference for the state of the art models provided by Mind and further support third-party models provided by users outside alibaba.
# Design doc # Design doc

View File

@@ -1 +1 @@
This folder will host example configs for each model supported by maas_lib. This folder will host example configs for each model supported by modelscope.

View File

@@ -1,34 +0,0 @@
maas\_lib.fileio.format package
===============================
.. automodule:: maas_lib.fileio.format
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
maas\_lib.fileio.format.base module
-----------------------------------
.. automodule:: maas_lib.fileio.format.base
:members:
:undoc-members:
:show-inheritance:
maas\_lib.fileio.format.json module
-----------------------------------
.. automodule:: maas_lib.fileio.format.json
:members:
:undoc-members:
:show-inheritance:
maas\_lib.fileio.format.yaml module
-----------------------------------
.. automodule:: maas_lib.fileio.format.yaml
:members:
:undoc-members:
:show-inheritance:

View File

@@ -1,34 +0,0 @@
maas\_lib.fileio package
========================
.. automodule:: maas_lib.fileio
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
:maxdepth: 4
maas_lib.fileio.format
Submodules
----------
maas\_lib.fileio.file module
----------------------------
.. automodule:: maas_lib.fileio.file
:members:
:undoc-members:
:show-inheritance:
maas\_lib.fileio.io module
--------------------------
.. automodule:: maas_lib.fileio.io
:members:
:undoc-members:
:show-inheritance:

View File

@@ -1,18 +0,0 @@
maas\_lib.models.nlp package
============================
.. automodule:: maas_lib.models.nlp
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
maas\_lib.models.nlp.sequence\_classification\_model module
-----------------------------------------------------------
.. automodule:: maas_lib.models.nlp.sequence_classification_model
:members:
:undoc-members:
:show-inheritance:

View File

@@ -1,34 +0,0 @@
maas\_lib.models package
========================
.. automodule:: maas_lib.models
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
:maxdepth: 4
maas_lib.models.nlp
Submodules
----------
maas\_lib.models.base module
----------------------------
.. automodule:: maas_lib.models.base
:members:
:undoc-members:
:show-inheritance:
maas\_lib.models.builder module
-------------------------------
.. automodule:: maas_lib.models.builder
:members:
:undoc-members:
:show-inheritance:

View File

@@ -1,7 +0,0 @@
maas\_lib.pipelines.audio package
=================================
.. automodule:: maas_lib.pipelines.audio
:members:
:undoc-members:
:show-inheritance:

View File

@@ -1,18 +0,0 @@
maas\_lib.pipelines.cv package
==============================
.. automodule:: maas_lib.pipelines.cv
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
maas\_lib.pipelines.cv.image\_matting module
--------------------------------------------
.. automodule:: maas_lib.pipelines.cv.image_matting
:members:
:undoc-members:
:show-inheritance:

View File

@@ -1,7 +0,0 @@
maas\_lib.pipelines.multi\_modal package
========================================
.. automodule:: maas_lib.pipelines.multi_modal
:members:
:undoc-members:
:show-inheritance:

View File

@@ -1,50 +0,0 @@
maas\_lib.preprocessors package
===============================
.. automodule:: maas_lib.preprocessors
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
maas\_lib.preprocessors.base module
-----------------------------------
.. automodule:: maas_lib.preprocessors.base
:members:
:undoc-members:
:show-inheritance:
maas\_lib.preprocessors.builder module
--------------------------------------
.. automodule:: maas_lib.preprocessors.builder
:members:
:undoc-members:
:show-inheritance:
maas\_lib.preprocessors.common module
-------------------------------------
.. automodule:: maas_lib.preprocessors.common
:members:
:undoc-members:
:show-inheritance:
maas\_lib.preprocessors.image module
------------------------------------
.. automodule:: maas_lib.preprocessors.image
:members:
:undoc-members:
:show-inheritance:
maas\_lib.preprocessors.nlp module
----------------------------------
.. automodule:: maas_lib.preprocessors.nlp
:members:
:undoc-members:
:show-inheritance:

View File

@@ -1,30 +0,0 @@
maas\_lib package
=================
.. automodule:: maas_lib
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
:maxdepth: 4
maas_lib.fileio
maas_lib.models
maas_lib.pipelines
maas_lib.preprocessors
maas_lib.utils
Submodules
----------
maas\_lib.version module
------------------------
.. automodule:: maas_lib.version
:members:
:undoc-members:
:show-inheritance:

View File

@@ -1,18 +0,0 @@
maas\_lib.trainers.nlp package
==============================
.. automodule:: maas_lib.trainers.nlp
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
maas\_lib.trainers.nlp.sequence\_classification\_trainer module
---------------------------------------------------------------
.. automodule:: maas_lib.trainers.nlp.sequence_classification_trainer
:members:
:undoc-members:
:show-inheritance:

View File

@@ -1,34 +0,0 @@
maas\_lib.trainers package
==========================
.. automodule:: maas_lib.trainers
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
:maxdepth: 4
maas_lib.trainers.nlp
Submodules
----------
maas\_lib.trainers.base module
------------------------------
.. automodule:: maas_lib.trainers.base
:members:
:undoc-members:
:show-inheritance:
maas\_lib.trainers.builder module
---------------------------------
.. automodule:: maas_lib.trainers.builder
:members:
:undoc-members:
:show-inheritance:

View File

@@ -1,58 +0,0 @@
maas\_lib.utils package
=======================
.. automodule:: maas_lib.utils
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
maas\_lib.utils.config module
-----------------------------
.. automodule:: maas_lib.utils.config
:members:
:undoc-members:
:show-inheritance:
maas\_lib.utils.constant module
-------------------------------
.. automodule:: maas_lib.utils.constant
:members:
:undoc-members:
:show-inheritance:
maas\_lib.utils.logger module
-----------------------------
.. automodule:: maas_lib.utils.logger
:members:
:undoc-members:
:show-inheritance:
maas\_lib.utils.pymod module
----------------------------
.. automodule:: maas_lib.utils.pymod
:members:
:undoc-members:
:show-inheritance:
maas\_lib.utils.registry module
-------------------------------
.. automodule:: maas_lib.utils.registry
:members:
:undoc-members:
:show-inheritance:
maas\_lib.utils.type\_assert module
-----------------------------------
.. automodule:: maas_lib.utils.type_assert
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,34 @@
modelscope.fileio.format package
================================
.. automodule:: modelscope.fileio.format
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
modelscope.fileio.format.base module
------------------------------------
.. automodule:: modelscope.fileio.format.base
:members:
:undoc-members:
:show-inheritance:
modelscope.fileio.format.json module
------------------------------------
.. automodule:: modelscope.fileio.format.json
:members:
:undoc-members:
:show-inheritance:
modelscope.fileio.format.yaml module
------------------------------------
.. automodule:: modelscope.fileio.format.yaml
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,34 @@
modelscope.fileio package
=========================
.. automodule:: modelscope.fileio
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
:maxdepth: 4
modelscope.fileio.format
Submodules
----------
modelscope.fileio.file module
-----------------------------
.. automodule:: modelscope.fileio.file
:members:
:undoc-members:
:show-inheritance:
modelscope.fileio.io module
---------------------------
.. automodule:: modelscope.fileio.io
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,18 @@
modelscope.models.cv.cartoon.facelib.LK package
===============================================
.. automodule:: modelscope.models.cv.cartoon.facelib.LK
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
modelscope.models.cv.cartoon.facelib.LK.lk module
-------------------------------------------------
.. automodule:: modelscope.models.cv.cartoon.facelib.LK.lk
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,50 @@
modelscope.models.cv.cartoon.facelib package
============================================
.. automodule:: modelscope.models.cv.cartoon.facelib
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
:maxdepth: 4
modelscope.models.cv.cartoon.facelib.LK
Submodules
----------
modelscope.models.cv.cartoon.facelib.config module
--------------------------------------------------
.. automodule:: modelscope.models.cv.cartoon.facelib.config
:members:
:undoc-members:
:show-inheritance:
modelscope.models.cv.cartoon.facelib.face\_detector module
----------------------------------------------------------
.. automodule:: modelscope.models.cv.cartoon.facelib.face_detector
:members:
:undoc-members:
:show-inheritance:
modelscope.models.cv.cartoon.facelib.face\_landmark module
----------------------------------------------------------
.. automodule:: modelscope.models.cv.cartoon.facelib.face_landmark
:members:
:undoc-members:
:show-inheritance:
modelscope.models.cv.cartoon.facelib.facer module
-------------------------------------------------
.. automodule:: modelscope.models.cv.cartoon.facelib.facer
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,15 @@
modelscope.models.cv.cartoon.mtcnn\_pytorch package
===================================================
.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
:maxdepth: 4
modelscope.models.cv.cartoon.mtcnn_pytorch.src

View File

@@ -0,0 +1,26 @@
modelscope.models.cv.cartoon.mtcnn\_pytorch.src package
=======================================================
.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch.src
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
modelscope.models.cv.cartoon.mtcnn\_pytorch.src.align\_trans module
-------------------------------------------------------------------
.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch.src.align_trans
:members:
:undoc-members:
:show-inheritance:
modelscope.models.cv.cartoon.mtcnn\_pytorch.src.matlab\_cp2tform module
-----------------------------------------------------------------------
.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch.src.matlab_cp2tform
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,27 @@
modelscope.models.cv.cartoon package
====================================
.. automodule:: modelscope.models.cv.cartoon
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
:maxdepth: 4
modelscope.models.cv.cartoon.facelib
modelscope.models.cv.cartoon.mtcnn_pytorch
Submodules
----------
modelscope.models.cv.cartoon.utils module
-----------------------------------------
.. automodule:: modelscope.models.cv.cartoon.utils
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,15 @@
modelscope.models.cv package
============================
.. automodule:: modelscope.models.cv
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
:maxdepth: 4
modelscope.models.cv.cartoon

View File

@@ -0,0 +1,26 @@
modelscope.models.nlp package
=============================
.. automodule:: modelscope.models.nlp
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
modelscope.models.nlp.sequence\_classification\_model module
------------------------------------------------------------
.. automodule:: modelscope.models.nlp.sequence_classification_model
:members:
:undoc-members:
:show-inheritance:
modelscope.models.nlp.text\_generation\_model module
----------------------------------------------------
.. automodule:: modelscope.models.nlp.text_generation_model
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,35 @@
modelscope.models package
=========================
.. automodule:: modelscope.models
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
:maxdepth: 4
modelscope.models.cv
modelscope.models.nlp
Submodules
----------
modelscope.models.base module
-----------------------------
.. automodule:: modelscope.models.base
:members:
:undoc-members:
:show-inheritance:
modelscope.models.builder module
--------------------------------
.. automodule:: modelscope.models.builder
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,7 @@
modelscope.pipelines.audio package
==================================
.. automodule:: modelscope.pipelines.audio
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,26 @@
modelscope.pipelines.cv package
===============================
.. automodule:: modelscope.pipelines.cv
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
modelscope.pipelines.cv.image\_cartoon\_pipeline module
-------------------------------------------------------
.. automodule:: modelscope.pipelines.cv.image_cartoon_pipeline
:members:
:undoc-members:
:show-inheritance:
modelscope.pipelines.cv.image\_matting\_pipeline module
-------------------------------------------------------
.. automodule:: modelscope.pipelines.cv.image_matting_pipeline
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,18 @@
modelscope.pipelines.multi\_modal package
=========================================
.. automodule:: modelscope.pipelines.multi_modal
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
modelscope.pipelines.multi\_modal.image\_captioning module
----------------------------------------------------------
.. automodule:: modelscope.pipelines.multi_modal.image_captioning
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,26 @@
modelscope.pipelines.nlp package
================================
.. automodule:: modelscope.pipelines.nlp
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
modelscope.pipelines.nlp.sequence\_classification\_pipeline module
------------------------------------------------------------------
.. automodule:: modelscope.pipelines.nlp.sequence_classification_pipeline
:members:
:undoc-members:
:show-inheritance:
modelscope.pipelines.nlp.text\_generation\_pipeline module
----------------------------------------------------------
.. automodule:: modelscope.pipelines.nlp.text_generation_pipeline
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,53 @@
modelscope.pipelines package
============================
.. automodule:: modelscope.pipelines
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
:maxdepth: 4
modelscope.pipelines.audio
modelscope.pipelines.cv
modelscope.pipelines.multi_modal
modelscope.pipelines.nlp
Submodules
----------
modelscope.pipelines.base module
--------------------------------
.. automodule:: modelscope.pipelines.base
:members:
:undoc-members:
:show-inheritance:
modelscope.pipelines.builder module
-----------------------------------
.. automodule:: modelscope.pipelines.builder
:members:
:undoc-members:
:show-inheritance:
modelscope.pipelines.default module
-----------------------------------
.. automodule:: modelscope.pipelines.default
:members:
:undoc-members:
:show-inheritance:
modelscope.pipelines.util module
--------------------------------
.. automodule:: modelscope.pipelines.util
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,50 @@
modelscope.preprocessors package
================================
.. automodule:: modelscope.preprocessors
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
modelscope.preprocessors.base module
------------------------------------
.. automodule:: modelscope.preprocessors.base
:members:
:undoc-members:
:show-inheritance:
modelscope.preprocessors.builder module
---------------------------------------
.. automodule:: modelscope.preprocessors.builder
:members:
:undoc-members:
:show-inheritance:
modelscope.preprocessors.common module
--------------------------------------
.. automodule:: modelscope.preprocessors.common
:members:
:undoc-members:
:show-inheritance:
modelscope.preprocessors.image module
-------------------------------------
.. automodule:: modelscope.preprocessors.image
:members:
:undoc-members:
:show-inheritance:
modelscope.preprocessors.nlp module
-----------------------------------
.. automodule:: modelscope.preprocessors.nlp
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,18 @@
modelscope.pydatasets package
=============================
.. automodule:: modelscope.pydatasets
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
modelscope.pydatasets.py\_dataset module
----------------------------------------
.. automodule:: modelscope.pydatasets.py_dataset
:members:
:undoc-members:
:show-inheritance:

View File

@@ -0,0 +1,32 @@
modelscope package
==================
.. automodule:: modelscope
:members:
:undoc-members:
:show-inheritance:
Subpackages
-----------
.. toctree::
:maxdepth: 4
modelscope.fileio
modelscope.models
modelscope.pipelines
modelscope.preprocessors
modelscope.pydatasets
modelscope.trainers
modelscope.utils
Submodules
----------
modelscope.version module
-------------------------
.. automodule:: modelscope.version
:members:
:undoc-members:
:show-inheritance:

View File

@@ -1,7 +1,7 @@
maas\_lib.pipelines.nlp package modelscope.trainers.nlp package
=============================== ===============================
.. automodule:: maas_lib.pipelines.nlp .. automodule:: modelscope.trainers.nlp
:members: :members:
:undoc-members: :undoc-members:
:show-inheritance: :show-inheritance:
@@ -9,10 +9,10 @@ maas\_lib.pipelines.nlp package
Submodules Submodules
---------- ----------
maas\_lib.pipelines.nlp.sequence\_classification\_pipeline module modelscope.trainers.nlp.sequence\_classification\_trainer module
----------------------------------------------------------------- ----------------------------------------------------------------
.. automodule:: maas_lib.pipelines.nlp.sequence_classification_pipeline .. automodule:: modelscope.trainers.nlp.sequence_classification_trainer
:members: :members:
:undoc-members: :undoc-members:
:show-inheritance: :show-inheritance:

View File

@@ -1,7 +1,7 @@
maas\_lib.pipelines package modelscope.trainers package
=========================== ===========================
.. automodule:: maas_lib.pipelines .. automodule:: modelscope.trainers
:members: :members:
:undoc-members: :undoc-members:
:show-inheritance: :show-inheritance:
@@ -12,25 +12,23 @@ Subpackages
.. toctree:: .. toctree::
:maxdepth: 4 :maxdepth: 4
maas_lib.pipelines.cv modelscope.trainers.nlp
maas_lib.pipelines.multi_modal
maas_lib.pipelines.nlp
Submodules Submodules
---------- ----------
maas\_lib.pipelines.base module modelscope.trainers.base module
------------------------------- -------------------------------
.. automodule:: maas_lib.pipelines.base .. automodule:: modelscope.trainers.base
:members: :members:
:undoc-members: :undoc-members:
:show-inheritance: :show-inheritance:
maas\_lib.pipelines.builder module modelscope.trainers.builder module
---------------------------------- ----------------------------------
.. automodule:: maas_lib.pipelines.builder .. automodule:: modelscope.trainers.builder
:members: :members:
:undoc-members: :undoc-members:
:show-inheritance: :show-inheritance:

View File

@@ -0,0 +1,66 @@
modelscope.utils package
========================
.. automodule:: modelscope.utils
:members:
:undoc-members:
:show-inheritance:
Submodules
----------
modelscope.utils.config module
------------------------------
.. automodule:: modelscope.utils.config
:members:
:undoc-members:
:show-inheritance:
modelscope.utils.constant module
--------------------------------
.. automodule:: modelscope.utils.constant
:members:
:undoc-members:
:show-inheritance:
modelscope.utils.hub module
---------------------------
.. automodule:: modelscope.utils.hub
:members:
:undoc-members:
:show-inheritance:
modelscope.utils.logger module
------------------------------
.. automodule:: modelscope.utils.logger
:members:
:undoc-members:
:show-inheritance:
modelscope.utils.pymod module
-----------------------------
.. automodule:: modelscope.utils.pymod
:members:
:undoc-members:
:show-inheritance:
modelscope.utils.registry module
--------------------------------
.. automodule:: modelscope.utils.registry
:members:
:undoc-members:
:show-inheritance:
modelscope.utils.type\_assert module
------------------------------------
.. automodule:: modelscope.utils.type_assert
:members:
:undoc-members:
:show-inheritance:

View File

@@ -1,7 +1,7 @@
maas_lib modelscope
======== ==========
.. toctree:: .. toctree::
:maxdepth: 4 :maxdepth: 4
maas_lib modelscope

View File

@@ -18,10 +18,10 @@ import sphinx_rtd_theme
sys.path.insert(0, os.path.abspath('../../')) sys.path.insert(0, os.path.abspath('../../'))
# -- Project information ----------------------------------------------------- # -- Project information -----------------------------------------------------
project = 'maas_lib' project = 'modelscope'
copyright = '2022-2023, Alibaba MaaS' copyright = '2022-2023, Alibaba ModelScope'
author = 'maas_lib Authors' author = 'modelscope Authors'
version_file = '../../maas_lib/version.py' version_file = '../../modelscope/version.py'
def get_version(): def get_version():
@@ -88,7 +88,7 @@ html_static_path = ['_static']
# -- Options for HTMLHelp output --------------------------------------------- # -- Options for HTMLHelp output ---------------------------------------------
# Output file base name for HTML help builder. # Output file base name for HTML help builder.
htmlhelp_basename = 'maas_lib_doc' htmlhelp_basename = 'modelscope_doc'
# -- Extension configuration ------------------------------------------------- # -- Extension configuration -------------------------------------------------
# Ignore >>> when copying code # Ignore >>> when copying code

View File

@@ -1,11 +1,11 @@
.. maas_lib documentation file, .. modelscope documentation file,
You can adapt this file completely to your liking, but it should at least You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive. contain the root `toctree` directive.
MaasLib DOCUMENTATION ModelScope DOCUMENTATION
======================================= =======================================
MaasLib doc ModelScope doc
.. toctree:: .. toctree::
:maxdepth: 2 :maxdepth: 2
@@ -30,11 +30,11 @@ MaasLib doc
:maxdepth: 10 :maxdepth: 10
:caption: API Doc :caption: API Doc
api/maas_lib.preprocessors api/modelscope.preprocessors
api/maas_lib.models api/modelscope.models
api/maas_lib.pipelines api/modelscope.pipelines
api/maas_lib.fileio api/modelscope.fileio
api/maas_lib.utils api/modelscope.utils
Indices and tables Indices and tables

View File

@@ -5,39 +5,39 @@
安装完成后执行如下命令为maas library创建对应的python环境。 安装完成后执行如下命令为maas library创建对应的python环境。
```shell ```shell
conda create -n maas python=3.6 conda create -n modelscope python=3.6
conda activate maas conda activate modelscope
``` ```
检查python和pip命令是否切换到conda环境下。 检查python和pip命令是否切换到conda环境下。
```shell ```shell
which python which python
# ~/workspace/anaconda3/envs/maas/bin/python # ~/workspace/anaconda3/envs/modelscope/bin/python
which pip which pip
# ~/workspace/anaconda3/envs/maas/bin/pip # ~/workspace/anaconda3/envs/modelscope/bin/pip
``` ```
注: 本项目只支持`python3`环境请勿使用python2环境。 注: 本项目只支持`python3`环境请勿使用python2环境。
## 第三方依赖安装 ## 第三方依赖安装
MaaS Library目前支持tensorflowpytorch两大深度学习框架进行模型训练、推理 在Python 3.6+, Pytorch 1.8+, Tensorflow 2.6上测试可运行,用户可以根据所选模型对应的计算框架进行安装,可以参考如下链接进行安装所需框架: ModelScope Library目前支持tensorflowpytorch两大深度学习框架进行模型训练、推理 在Python 3.6+, Pytorch 1.8+, Tensorflow 2.6上测试可运行,用户可以根据所选模型对应的计算框架进行安装,可以参考如下链接进行安装所需框架:
* [Pytorch安装指导](https://pytorch.org/get-started/locally/) * [Pytorch安装指导](https://pytorch.org/get-started/locally/)
* [Tensorflow安装指导](https://www.tensorflow.org/install/pip) * [Tensorflow安装指导](https://www.tensorflow.org/install/pip)
## MaaS library 安装 ## ModelScope library 安装
注: 如果在安装过程中遇到错误,请前往[常见问题](faq.md)查找解决方案。 注: 如果在安装过程中遇到错误,请前往[常见问题](faq.md)查找解决方案。
### pip安装 ### pip安装
```shell ```shell
pip install -r http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/release/maas/maas.txt pip install -r http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/release/maas/modelscope.txt
``` ```
安装成功后,可以执行如下命令进行验证安装是否正确 安装成功后,可以执行如下命令进行验证安装是否正确
```shell ```shell
python -c "from maas_lib.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))" python -c "from modelscope.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))"
``` ```
@@ -45,11 +45,11 @@ python -c "from maas_lib.pipelines import pipeline;print(pipeline('image-matting
适合本地开发调试使用,修改源码后可以直接执行 适合本地开发调试使用,修改源码后可以直接执行
```shell ```shell
git clone git@gitlab.alibaba-inc.com:Ali-MaaS/MaaS-lib.git maaslib git clone git@gitlab.alibaba-inc.com:Ali-MaaS/MaaS-lib.git modelscope
git fetch origin master git fetch origin master
git checkout master git checkout master
cd maaslib cd modelscope
#安装依赖 #安装依赖
pip install -r requirements.txt pip install -r requirements.txt
@@ -60,7 +60,7 @@ export PYTHONPATH=`pwd`
安装成功后,可以执行如下命令进行验证安装是否正确 安装成功后,可以执行如下命令进行验证安装是否正确
```shell ```shell
python -c "from maas_lib.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))" python -c "from modelscope.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))"
``` ```
@@ -79,8 +79,8 @@ pipeline函数提供了简洁的推理接口示例如下 更多pipeline介
```python ```python
import cv2 import cv2
import os.path as osp import os.path as osp
from maas_lib.pipelines import pipeline from modelscope.pipelines import pipeline
from maas_lib.utils.constant import Tasks from modelscope.utils.constant import Tasks
# 根据任务名创建pipeline # 根据任务名创建pipeline
img_matting = pipeline(Tasks.image_matting, model='damo/image-matting-person') img_matting = pipeline(Tasks.image_matting, model='damo/image-matting-person')
@@ -99,9 +99,9 @@ print(f'Output written to {osp.abspath("result.png")}')
```python ```python
import cv2 import cv2
import os.path as osp import os.path as osp
from maas_lib.pipelines import pipeline from modelscope.pipelines import pipeline
from maas_lib.utils.constant import Tasks from modelscope.utils.constant import Tasks
from maas_lib.pydatasets import PyDataset from modelscope.pydatasets import PyDataset
# 使用图像url构建PyDataset此处也可通过 input_location = '/dir/to/images' 来使用本地文件夹 # 使用图像url构建PyDataset此处也可通过 input_location = '/dir/to/images' 来使用本地文件夹
input_location = [ input_location = [

View File

@@ -19,7 +19,7 @@
1. pipeline函数支持指定特定任务名称加载任务默认模型创建对应Pipeline对象 1. pipeline函数支持指定特定任务名称加载任务默认模型创建对应Pipeline对象
执行如下python代码 执行如下python代码
```python ```python
>>> from maas_lib.pipelines import pipeline >>> from modelscope.pipelines import pipeline
>>> img_matting = pipeline(task='image-matting', model='damo/image-matting-person') >>> img_matting = pipeline(task='image-matting', model='damo/image-matting-person')
``` ```
@@ -65,8 +65,8 @@ wget https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/easynlp_modelz
创建tokenizer和模型 创建tokenizer和模型
```python ```python
>>> from maas_lib.models import Model >>> from modelscope.models import Model
>>> from maas_lib.preprocessors import SequenceClassificationPreprocessor >>> from modelscope.preprocessors import SequenceClassificationPreprocessor
>>> model = Model.from_pretrained('damo/bert-base-sst2') >>> model = Model.from_pretrained('damo/bert-base-sst2')
>>> tokenizer = SequenceClassificationPreprocessor( >>> tokenizer = SequenceClassificationPreprocessor(
model.model_dir, first_sequence='sentence', second_sequence=None) model.model_dir, first_sequence='sentence', second_sequence=None)
@@ -74,7 +74,7 @@ wget https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/easynlp_modelz
使用tokenizer和模型对象创建pipeline 使用tokenizer和模型对象创建pipeline
```python ```python
>>> from maas_lib.pipelines import pipeline >>> from modelscope.pipelines import pipeline
>>> semantic_cls = pipeline('text-classification', model=model, preprocessor=tokenizer) >>> semantic_cls = pipeline('text-classification', model=model, preprocessor=tokenizer)
>>> semantic_cls("Hello world!") >>> semantic_cls("Hello world!")
``` ```

View File

@@ -1,48 +0,0 @@
# Copyright (c) Alibaba, Inc. and its affiliates.
from maas_lib.utils.constant import Tasks
DEFAULT_MODEL_FOR_PIPELINE = {
# TaskName: (pipeline_module_name, model_repo)
Tasks.image_matting: ('image-matting', 'damo/image-matting-person'),
Tasks.text_classification:
('bert-sentiment-analysis', 'damo/bert-base-sst2'),
Tasks.text_generation: ('palm', 'damo/nlp_palm_text-generation_chinese'),
Tasks.image_captioning: ('ofa', None),
}
def add_default_pipeline_info(task: str,
model_name: str,
modelhub_name: str = None,
overwrite: bool = False):
""" Add default model for a task.
Args:
task (str): task name.
model_name (str): model_name.
modelhub_name (str): name for default modelhub.
overwrite (bool): overwrite default info.
"""
if not overwrite:
assert task not in DEFAULT_MODEL_FOR_PIPELINE, \
f'task {task} already has default model.'
DEFAULT_MODEL_FOR_PIPELINE[task] = (model_name, modelhub_name)
def get_default_pipeline_info(task):
""" Get default info for certain task.
Args:
task (str): task name.
Return:
A tuple: first element is pipeline name(model_name), second element
is modelhub name.
"""
assert task in DEFAULT_MODEL_FOR_PIPELINE, \
f'No default pipeline is registered for Task {task}'
pipeline_name, default_model = DEFAULT_MODEL_FOR_PIPELINE[task]
return pipeline_name, default_model

View File

@@ -7,10 +7,10 @@ from typing import Dict, List, Tuple, Union
from maas_hub.file_download import model_file_download from maas_hub.file_download import model_file_download
from maas_hub.snapshot_download import snapshot_download from maas_hub.snapshot_download import snapshot_download
from maas_lib.models.builder import build_model from modelscope.models.builder import build_model
from maas_lib.utils.config import Config from modelscope.utils.config import Config
from maas_lib.utils.constant import CONFIGFILE from modelscope.utils.constant import CONFIGFILE
from maas_lib.utils.hub import get_model_cache_dir from modelscope.utils.hub import get_model_cache_dir
Tensor = Union['torch.Tensor', 'tf.Tensor'] Tensor = Union['torch.Tensor', 'tf.Tensor']

View File

@@ -1,7 +1,7 @@
# Copyright (c) Alibaba, Inc. and its affiliates. # Copyright (c) Alibaba, Inc. and its affiliates.
from maas_lib.utils.config import ConfigDict from modelscope.utils.config import ConfigDict
from maas_lib.utils.registry import Registry, build_from_cfg from modelscope.utils.registry import Registry, build_from_cfg
MODELS = Registry('models') MODELS = Registry('models')

View File

@@ -2,7 +2,7 @@ from typing import Any, Dict
import numpy as np import numpy as np
from maas_lib.utils.constant import Tasks from modelscope.utils.constant import Tasks
from ..base import Model from ..base import Model
from ..builder import MODELS from ..builder import MODELS

View File

@@ -1,6 +1,6 @@
from typing import Any, Dict from typing import Any, Dict
from maas_lib.utils.constant import Tasks from modelscope.utils.constant import Tasks
from ..base import Model, Tensor from ..base import Model, Tensor
from ..builder import MODELS from ..builder import MODELS

View File

@@ -6,11 +6,11 @@ from typing import Any, Dict, Generator, List, Union
from maas_hub.snapshot_download import snapshot_download from maas_hub.snapshot_download import snapshot_download
from maas_lib.models.base import Model from modelscope.models.base import Model
from maas_lib.preprocessors import Preprocessor from modelscope.preprocessors import Preprocessor
from maas_lib.pydatasets import PyDataset from modelscope.pydatasets import PyDataset
from maas_lib.utils.config import Config from modelscope.utils.config import Config
from maas_lib.utils.hub import get_model_cache_dir from modelscope.utils.hub import get_model_cache_dir
from .util import is_model_name from .util import is_model_name
Tensor = Union['torch.Tensor', 'tf.Tensor'] Tensor = Union['torch.Tensor', 'tf.Tensor']
@@ -75,7 +75,7 @@ class Pipeline(ABC):
def __call__(self, input: Union[Input, List[Input]], *args, def __call__(self, input: Union[Input, List[Input]], *args,
**post_kwargs) -> Union[Dict[str, Any], Generator]: **post_kwargs) -> Union[Dict[str, Any], Generator]:
# model provider should leave it as it is # model provider should leave it as it is
# maas library developer will handle this function # modelscope library developer will handle this function
# simple showcase, need to support iterator type for both tensorflow and pytorch # simple showcase, need to support iterator type for both tensorflow and pytorch
# input_dict = self._handle_input(input) # input_dict = self._handle_input(input)

View File

@@ -6,16 +6,26 @@ from typing import Union
import json import json
from maas_hub.file_download import model_file_download from maas_hub.file_download import model_file_download
from maas_lib.models.base import Model from modelscope.models.base import Model
from maas_lib.utils.config import Config, ConfigDict from modelscope.utils.config import Config, ConfigDict
from maas_lib.utils.constant import CONFIGFILE, Tasks from modelscope.utils.constant import CONFIGFILE, Tasks
from maas_lib.utils.registry import Registry, build_from_cfg from modelscope.utils.registry import Registry, build_from_cfg
from .base import Pipeline from .base import Pipeline
from .default import DEFAULT_MODEL_FOR_PIPELINE, get_default_pipeline_info
from .util import is_model_name from .util import is_model_name
PIPELINES = Registry('pipelines') PIPELINES = Registry('pipelines')
DEFAULT_MODEL_FOR_PIPELINE = {
# TaskName: (pipeline_module_name, model_repo)
Tasks.image_matting: ('image-matting', 'damo/image-matting-person'),
Tasks.text_classification:
('bert-sentiment-analysis', 'damo/bert-base-sst2'),
Tasks.text_generation: ('palm', 'damo/nlp_palm_text-generation_chinese'),
Tasks.image_captioning: ('ofa', None),
Tasks.image_generation:
('cv_unet_person-image-cartoon', 'damo/cv_unet_image-matting_damo'),
}
def build_pipeline(cfg: ConfigDict, def build_pipeline(cfg: ConfigDict,
task_name: str = None, task_name: str = None,
@@ -84,3 +94,42 @@ def pipeline(task: str = None,
cfg.preprocessor = preprocessor cfg.preprocessor = preprocessor
return build_pipeline(cfg, task_name=task) return build_pipeline(cfg, task_name=task)
def add_default_pipeline_info(task: str,
model_name: str,
modelhub_name: str = None,
overwrite: bool = False):
""" Add default model for a task.
Args:
task (str): task name.
model_name (str): model_name.
modelhub_name (str): name for default modelhub.
overwrite (bool): overwrite default info.
"""
if not overwrite:
assert task not in DEFAULT_MODEL_FOR_PIPELINE, \
f'task {task} already has default model.'
DEFAULT_MODEL_FOR_PIPELINE[task] = (model_name, modelhub_name)
def get_default_pipeline_info(task):
""" Get default info for certain task.
Args:
task (str): task name.
Return:
A tuple: first element is pipeline name(model_name), second element
is modelhub name.
"""
if task not in DEFAULT_MODEL_FOR_PIPELINE:
# support pipeline which does not register default model
pipeline_name = list(PIPELINES.modules[task].keys())[0]
default_model = None
else:
pipeline_name, default_model = DEFAULT_MODEL_FOR_PIPELINE[task]
return pipeline_name, default_model

View File

@@ -6,14 +6,14 @@ import numpy as np
import PIL import PIL
import tensorflow as tf import tensorflow as tf
from maas_lib.models.cv.cartoon.facelib.facer import FaceAna from modelscope.models.cv.cartoon.facelib.facer import FaceAna
from maas_lib.models.cv.cartoon.mtcnn_pytorch.src.align_trans import ( from modelscope.models.cv.cartoon.mtcnn_pytorch.src.align_trans import (
get_reference_facial_points, warp_and_crop_face) get_reference_facial_points, warp_and_crop_face)
from maas_lib.models.cv.cartoon.utils import get_f5p, padTo16x, resize_size from modelscope.models.cv.cartoon.utils import get_f5p, padTo16x, resize_size
from maas_lib.pipelines.base import Input from modelscope.pipelines.base import Input
from maas_lib.preprocessors import load_image from modelscope.preprocessors import load_image
from maas_lib.utils.constant import Tasks from modelscope.utils.constant import Tasks
from maas_lib.utils.logger import get_logger from modelscope.utils.logger import get_logger
from ..base import Pipeline from ..base import Pipeline
from ..builder import PIPELINES from ..builder import PIPELINES

View File

@@ -5,10 +5,10 @@ import cv2
import numpy as np import numpy as np
import PIL import PIL
from maas_lib.pipelines.base import Input from modelscope.pipelines.base import Input
from maas_lib.preprocessors import load_image from modelscope.preprocessors import load_image
from maas_lib.utils.constant import Tasks from modelscope.utils.constant import Tasks
from maas_lib.utils.logger import get_logger from modelscope.utils.logger import get_logger
from ..base import Pipeline from ..base import Pipeline
from ..builder import PIPELINES from ..builder import PIPELINES

View File

@@ -4,10 +4,10 @@ import numpy as np
import torch import torch
from PIL import Image from PIL import Image
from maas_lib.pipelines.base import Input from modelscope.pipelines.base import Input
from maas_lib.preprocessors import load_image from modelscope.preprocessors import load_image
from maas_lib.utils.constant import Tasks from modelscope.utils.constant import Tasks
from maas_lib.utils.logger import get_logger from modelscope.utils.logger import get_logger
from ..base import Pipeline from ..base import Pipeline
from ..builder import PIPELINES from ..builder import PIPELINES

View File

@@ -5,9 +5,9 @@ from typing import Any, Dict, Union
import json import json
import numpy as np import numpy as np
from maas_lib.models.nlp import BertForSequenceClassification from modelscope.models.nlp import BertForSequenceClassification
from maas_lib.preprocessors import SequenceClassificationPreprocessor from modelscope.preprocessors import SequenceClassificationPreprocessor
from maas_lib.utils.constant import Tasks from modelscope.utils.constant import Tasks
from ...models import Model from ...models import Model
from ..base import Input, Pipeline from ..base import Input, Pipeline
from ..builder import PIPELINES from ..builder import PIPELINES

View File

@@ -1,9 +1,9 @@
from typing import Dict, Optional, Union from typing import Dict, Optional, Union
from maas_lib.models import Model from modelscope.models import Model
from maas_lib.models.nlp import PalmForTextGenerationModel from modelscope.models.nlp import PalmForTextGenerationModel
from maas_lib.preprocessors import TextGenerationPreprocessor from modelscope.preprocessors import TextGenerationPreprocessor
from maas_lib.utils.constant import Tasks from modelscope.utils.constant import Tasks
from ..base import Pipeline, Tensor from ..base import Pipeline, Tensor
from ..builder import PIPELINES from ..builder import PIPELINES

View File

@@ -5,7 +5,7 @@ import os.path as osp
import json import json
from maas_hub.file_download import model_file_download from maas_hub.file_download import model_file_download
from maas_lib.utils.constant import CONFIGFILE from modelscope.utils.constant import CONFIGFILE
def is_model_name(model): def is_model_name(model):

View File

@@ -1,8 +1,8 @@
# Copyright (c) Alibaba, Inc. and its affiliates. # Copyright (c) Alibaba, Inc. and its affiliates.
from maas_lib.utils.config import ConfigDict from modelscope.utils.config import ConfigDict
from maas_lib.utils.constant import Fields from modelscope.utils.constant import Fields
from maas_lib.utils.registry import Registry, build_from_cfg from modelscope.utils.registry import Registry, build_from_cfg
PREPROCESSORS = Registry('preprocessors') PREPROCESSORS = Registry('preprocessors')

View File

@@ -4,8 +4,8 @@ from typing import Dict, Union
from PIL import Image, ImageOps from PIL import Image, ImageOps
from maas_lib.fileio import File from modelscope.fileio import File
from maas_lib.utils.constant import Fields from modelscope.utils.constant import Fields
from .builder import PREPROCESSORS from .builder import PREPROCESSORS

View File

@@ -5,8 +5,8 @@ from typing import Any, Dict, Union
from transformers import AutoTokenizer from transformers import AutoTokenizer
from maas_lib.utils.constant import Fields, InputFields from modelscope.utils.constant import Fields, InputFields
from maas_lib.utils.type_assert import type_assert from modelscope.utils.type_assert import type_assert
from .base import Preprocessor from .base import Preprocessor
from .builder import PREPROCESSORS from .builder import PREPROCESSORS

View File

@@ -4,7 +4,7 @@ from typing import (Any, Callable, Dict, List, Mapping, Optional, Sequence,
from datasets import Dataset, load_dataset from datasets import Dataset, load_dataset
from maas_lib.utils.logger import get_logger from modelscope.utils.logger import get_logger
logger = get_logger() logger = get_logger()
@@ -52,7 +52,7 @@ class PyDataset:
Mapping[str, Union[str, Mapping[str, Union[str,
Sequence[str]]]]] = None Sequence[str]]]]] = None
) -> 'PyDataset': ) -> 'PyDataset':
"""Load a PyDataset from the MaaS Hub, Hugging Face Hub, urls, or a local dataset. """Load a PyDataset from the ModelScope Hub, Hugging Face Hub, urls, or a local dataset.
Args: Args:
path (str): Path or name of the dataset. path (str): Path or name of the dataset.

View File

@@ -2,7 +2,7 @@
import argparse import argparse
from maas_lib.trainers import build_trainer from modelscope.trainers import build_trainer
def parse_args(): def parse_args():

View File

@@ -2,7 +2,7 @@
import argparse import argparse
from maas_lib.trainers import build_trainer from modelscope.trainers import build_trainer
def parse_args(): def parse_args():

Some files were not shown because too many files have changed in this diff Show More