From db4a8be9c5c7ee377b5f6e8070cf8da710c9fcd5 Mon Sep 17 00:00:00 2001 From: "wenmeng.zwm" Date: Fri, 20 May 2022 16:51:34 +0800 Subject: [PATCH] [to #41669377] docs and tools refinement and release 1. add build_doc linter script 2. add sphinx-docs support 3. add development doc and api doc 4. change version to 0.1.0 for the first internal release version Link: https://code.aone.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/8775307 --- .dev_scripts/build_docs.sh | 8 + .dev_scripts/linter.sh | 3 + .readthedocs.yaml | 28 +++ MANIFEST.in | 1 + Makefile | 25 +++ README.md | 9 + configs/examples/config.py | 2 +- docs/Makefile | 20 ++ docs/README.md | 43 ++++ docs/make.bat | 35 ++++ docs/source/api/maas_lib.fileio.format.rst | 34 ++++ docs/source/api/maas_lib.fileio.rst | 34 ++++ docs/source/api/maas_lib.models.nlp.rst | 18 ++ docs/source/api/maas_lib.models.rst | 34 ++++ docs/source/api/maas_lib.pipelines.cv.rst | 18 ++ .../api/maas_lib.pipelines.multi_modal.rst | 7 + docs/source/api/maas_lib.pipelines.nlp.rst | 18 ++ docs/source/api/maas_lib.pipelines.rst | 36 ++++ docs/source/api/maas_lib.preprocessors.rst | 50 +++++ docs/source/api/maas_lib.rst | 30 +++ docs/source/api/maas_lib.utils.rst | 58 ++++++ docs/source/api/modules.rst | 7 + docs/source/change_log.md | 13 ++ docs/source/conf.py | 104 ++++++++++ docs/source/develop.md | 48 +++++ docs/source/index.rst | 43 ++++ docs/source/tutorials/index.rst | 3 + maas_lib/fileio/file.py | 8 +- maas_lib/fileio/io.py | 4 +- .../nlp/sequence_classification_model.py | 6 +- maas_lib/pipelines/builder.py | 4 +- maas_lib/preprocessors/image.py | 2 +- maas_lib/version.py | 2 +- setup.py | 183 ++++++++++++++++++ 34 files changed, 924 insertions(+), 14 deletions(-) create mode 100644 .dev_scripts/build_docs.sh create mode 100644 .dev_scripts/linter.sh create mode 100644 .readthedocs.yaml create mode 100644 MANIFEST.in create mode 100644 Makefile create mode 100644 docs/Makefile create mode 100644 docs/README.md create mode 100644 docs/make.bat create mode 100644 docs/source/api/maas_lib.fileio.format.rst create mode 100644 docs/source/api/maas_lib.fileio.rst create mode 100644 docs/source/api/maas_lib.models.nlp.rst create mode 100644 docs/source/api/maas_lib.models.rst create mode 100644 docs/source/api/maas_lib.pipelines.cv.rst create mode 100644 docs/source/api/maas_lib.pipelines.multi_modal.rst create mode 100644 docs/source/api/maas_lib.pipelines.nlp.rst create mode 100644 docs/source/api/maas_lib.pipelines.rst create mode 100644 docs/source/api/maas_lib.preprocessors.rst create mode 100644 docs/source/api/maas_lib.rst create mode 100644 docs/source/api/maas_lib.utils.rst create mode 100644 docs/source/api/modules.rst create mode 100644 docs/source/change_log.md create mode 100644 docs/source/conf.py create mode 100644 docs/source/develop.md create mode 100644 docs/source/index.rst create mode 100644 docs/source/tutorials/index.rst create mode 100644 setup.py diff --git a/.dev_scripts/build_docs.sh b/.dev_scripts/build_docs.sh new file mode 100644 index 00000000..9c8acdf1 --- /dev/null +++ b/.dev_scripts/build_docs.sh @@ -0,0 +1,8 @@ +pip install -r requirements/docs.txt +cd docs +rm -rf build + +# update api rst +#rm -rf source/api/ +#sphinx-apidoc --module-first -o source/api/ ../maas_lib/ +make html diff --git a/.dev_scripts/linter.sh b/.dev_scripts/linter.sh new file mode 100644 index 00000000..fb8ab19d --- /dev/null +++ b/.dev_scripts/linter.sh @@ -0,0 +1,3 @@ +yapf -r -i maas_lib/ configs/ tests/ setup.py +isort -rc maas_lib/ configs/ tests/ setup.py +flake8 maas_lib/ configs/ tests/ setup.py diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..b88d734a --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,28 @@ +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-20.04 + tools: + python: "3.7" + # You can also specify other tool versions: + # nodejs: "16" + # rust: "1.55" + # golang: "1.17" + jobs: + post_checkout: + - echo "dummy" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# If using Sphinx, optionally build your docs in additional formats such as PDF +# formats: +formats: all + +python: + install: + - requirements: requirements/docs.txt + - requirements: requirements/readthedocs.txt + - requirements: requirements/runtime.txt diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..0a153dba --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +recursive-include maas_lib/configs *.py diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..96532199 --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +WHL_BUILD_DIR :=package +DOC_BUILD_DIR :=docs/build/ + +# default rule +default: whl docs + +.PHONY: docs +docs: + bash .dev_scripts/build_docs.sh + +.PHONY: linter +linter: + bash .dev_scripts/linter.sh + +.PHONY: test +test: + bash .dev_scripts/citest.sh + +.PHONY: whl +whl: + python setup.py sdist bdist_wheel + +.PHONY: clean +clean: + rm -rf $(WHL_BUILD_DIR) $(DOC_BUILD_DIR) diff --git a/README.md b/README.md index 9f1487d0..dabe8726 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,12 @@ MaaS library is targeted to support training, evaluation and inference for the s # Design doc Please refer to alidoc [link](https://alidocs.dingtalk.com/i/nodes/OBldywvrKxo89xmAO05yJQk2ngpNbLz4?nav=spaces&navQuery=spaceId%3Dnb9XJNlZxbgrOXyA&iframeQuery=utm_source%3Dportal%26utm_medium%3Dportal_space_file_tree) + +# Development doc + +Please refer to [develop.md](docs/source/develop.md) + +# ChangeLog +* 20/05/2022 First release version + +Refer to [change_log.md](docs/source/change_log.md) for more details diff --git a/configs/examples/config.py b/configs/examples/config.py index aab2c3c5..beafb8ee 100644 --- a/configs/examples/config.py +++ b/configs/examples/config.py @@ -1,2 +1,2 @@ a = 1 -b = dict(c=[1,2,3], d='dd') +b = dict(c=[1, 2, 3], d='dd') diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..d0c3cbf1 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..e8b71575 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,43 @@ +## maintain docs +1. install requirements needed to build docs + ```shell + # in maas_lib root dir + pip install requirements/docs.txt + ``` + +2. build docs + ```shell + # in maas_lib/docs dir + bash build_docs.sh + ``` + +3. doc string format + + We adopt the google style docstring format as the standard, please refer to the following documents. + 1. Google Python style guide docstring [link](http://google.github.io/styleguide/pyguide.html#381-docstrings) + 2. Google docstring example [link](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) + 3. sample：torch.nn.modules.conv [link](https://pytorch.org/docs/stable/_modules/torch/nn/modules/conv.html#Conv1d) + 4. load fucntion as an example： + + ```python + def load(file, file_format=None, **kwargs): + """Load data from json/yaml/pickle files. + + This method provides a unified api for loading data from serialized files. + + Args: + file (str or :obj:`Path` or file-like object): Filename or a file-like + object. + file_format (str, optional): If not specified, the file format will be + inferred from the file extension, otherwise use the specified one. + Currently supported formats include "json", "yaml/yml". + + Examples: + >>> load('/path/of/your/file') # file is storaged in disk + >>> load('https://path/of/your/file') # file is storaged in Internet + >>> load('oss://path/of/your/file') # file is storaged in petrel + + Returns: + The content from the file. + """ + ``` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..3d64bb3a --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/api/maas_lib.fileio.format.rst b/docs/source/api/maas_lib.fileio.format.rst new file mode 100644 index 00000000..7c2c649d --- /dev/null +++ b/docs/source/api/maas_lib.fileio.format.rst @@ -0,0 +1,34 @@ +maas\_lib.fileio.format package +=============================== + +.. automodule:: maas_lib.fileio.format + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +maas\_lib.fileio.format.base module +----------------------------------- + +.. automodule:: maas_lib.fileio.format.base + :members: + :undoc-members: + :show-inheritance: + +maas\_lib.fileio.format.json module +----------------------------------- + +.. automodule:: maas_lib.fileio.format.json + :members: + :undoc-members: + :show-inheritance: + +maas\_lib.fileio.format.yaml module +----------------------------------- + +.. automodule:: maas_lib.fileio.format.yaml + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/maas_lib.fileio.rst b/docs/source/api/maas_lib.fileio.rst new file mode 100644 index 00000000..e9540208 --- /dev/null +++ b/docs/source/api/maas_lib.fileio.rst @@ -0,0 +1,34 @@ +maas\_lib.fileio package +======================== + +.. automodule:: maas_lib.fileio + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + maas_lib.fileio.format + +Submodules +---------- + +maas\_lib.fileio.file module +---------------------------- + +.. automodule:: maas_lib.fileio.file + :members: + :undoc-members: + :show-inheritance: + +maas\_lib.fileio.io module +-------------------------- + +.. automodule:: maas_lib.fileio.io + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/maas_lib.models.nlp.rst b/docs/source/api/maas_lib.models.nlp.rst new file mode 100644 index 00000000..bd782ea8 --- /dev/null +++ b/docs/source/api/maas_lib.models.nlp.rst @@ -0,0 +1,18 @@ +maas\_lib.models.nlp package +============================ + +.. automodule:: maas_lib.models.nlp + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +maas\_lib.models.nlp.sequence\_classification\_model module +----------------------------------------------------------- + +.. automodule:: maas_lib.models.nlp.sequence_classification_model + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/maas_lib.models.rst b/docs/source/api/maas_lib.models.rst new file mode 100644 index 00000000..9e1874a3 --- /dev/null +++ b/docs/source/api/maas_lib.models.rst @@ -0,0 +1,34 @@ +maas\_lib.models package +======================== + +.. automodule:: maas_lib.models + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + maas_lib.models.nlp + +Submodules +---------- + +maas\_lib.models.base module +---------------------------- + +.. automodule:: maas_lib.models.base + :members: + :undoc-members: + :show-inheritance: + +maas\_lib.models.builder module +------------------------------- + +.. automodule:: maas_lib.models.builder + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/maas_lib.pipelines.cv.rst b/docs/source/api/maas_lib.pipelines.cv.rst new file mode 100644 index 00000000..938ebb5a --- /dev/null +++ b/docs/source/api/maas_lib.pipelines.cv.rst @@ -0,0 +1,18 @@ +maas\_lib.pipelines.cv package +============================== + +.. automodule:: maas_lib.pipelines.cv + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +maas\_lib.pipelines.cv.image\_matting module +-------------------------------------------- + +.. automodule:: maas_lib.pipelines.cv.image_matting + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/maas_lib.pipelines.multi_modal.rst b/docs/source/api/maas_lib.pipelines.multi_modal.rst new file mode 100644 index 00000000..74a7bf43 --- /dev/null +++ b/docs/source/api/maas_lib.pipelines.multi_modal.rst @@ -0,0 +1,7 @@ +maas\_lib.pipelines.multi\_modal package +======================================== + +.. automodule:: maas_lib.pipelines.multi_modal + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/maas_lib.pipelines.nlp.rst b/docs/source/api/maas_lib.pipelines.nlp.rst new file mode 100644 index 00000000..d41c09ad --- /dev/null +++ b/docs/source/api/maas_lib.pipelines.nlp.rst @@ -0,0 +1,18 @@ +maas\_lib.pipelines.nlp package +=============================== + +.. automodule:: maas_lib.pipelines.nlp + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +maas\_lib.pipelines.nlp.sequence\_classification\_pipeline module +----------------------------------------------------------------- + +.. automodule:: maas_lib.pipelines.nlp.sequence_classification_pipeline + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/maas_lib.pipelines.rst b/docs/source/api/maas_lib.pipelines.rst new file mode 100644 index 00000000..40b82adc --- /dev/null +++ b/docs/source/api/maas_lib.pipelines.rst @@ -0,0 +1,36 @@ +maas\_lib.pipelines package +=========================== + +.. automodule:: maas_lib.pipelines + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + maas_lib.pipelines.cv + maas_lib.pipelines.multi_modal + maas_lib.pipelines.nlp + +Submodules +---------- + +maas\_lib.pipelines.base module +------------------------------- + +.. automodule:: maas_lib.pipelines.base + :members: + :undoc-members: + :show-inheritance: + +maas\_lib.pipelines.builder module +---------------------------------- + +.. automodule:: maas_lib.pipelines.builder + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/maas_lib.preprocessors.rst b/docs/source/api/maas_lib.preprocessors.rst new file mode 100644 index 00000000..5f70e808 --- /dev/null +++ b/docs/source/api/maas_lib.preprocessors.rst @@ -0,0 +1,50 @@ +maas\_lib.preprocessors package +=============================== + +.. automodule:: maas_lib.preprocessors + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +maas\_lib.preprocessors.base module +----------------------------------- + +.. automodule:: maas_lib.preprocessors.base + :members: + :undoc-members: + :show-inheritance: + +maas\_lib.preprocessors.builder module +-------------------------------------- + +.. automodule:: maas_lib.preprocessors.builder + :members: + :undoc-members: + :show-inheritance: + +maas\_lib.preprocessors.common module +------------------------------------- + +.. automodule:: maas_lib.preprocessors.common + :members: + :undoc-members: + :show-inheritance: + +maas\_lib.preprocessors.image module +------------------------------------ + +.. automodule:: maas_lib.preprocessors.image + :members: + :undoc-members: + :show-inheritance: + +maas\_lib.preprocessors.nlp module +---------------------------------- + +.. automodule:: maas_lib.preprocessors.nlp + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/maas_lib.rst b/docs/source/api/maas_lib.rst new file mode 100644 index 00000000..727b7986 --- /dev/null +++ b/docs/source/api/maas_lib.rst @@ -0,0 +1,30 @@ +maas\_lib package +================= + +.. automodule:: maas_lib + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + maas_lib.fileio + maas_lib.models + maas_lib.pipelines + maas_lib.preprocessors + maas_lib.utils + +Submodules +---------- + +maas\_lib.version module +------------------------ + +.. automodule:: maas_lib.version + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/maas_lib.utils.rst b/docs/source/api/maas_lib.utils.rst new file mode 100644 index 00000000..17ead3eb --- /dev/null +++ b/docs/source/api/maas_lib.utils.rst @@ -0,0 +1,58 @@ +maas\_lib.utils package +======================= + +.. automodule:: maas_lib.utils + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +maas\_lib.utils.config module +----------------------------- + +.. automodule:: maas_lib.utils.config + :members: + :undoc-members: + :show-inheritance: + +maas\_lib.utils.constant module +------------------------------- + +.. automodule:: maas_lib.utils.constant + :members: + :undoc-members: + :show-inheritance: + +maas\_lib.utils.logger module +----------------------------- + +.. automodule:: maas_lib.utils.logger + :members: + :undoc-members: + :show-inheritance: + +maas\_lib.utils.pymod module +---------------------------- + +.. automodule:: maas_lib.utils.pymod + :members: + :undoc-members: + :show-inheritance: + +maas\_lib.utils.registry module +------------------------------- + +.. automodule:: maas_lib.utils.registry + :members: + :undoc-members: + :show-inheritance: + +maas\_lib.utils.type\_assert module +----------------------------------- + +.. automodule:: maas_lib.utils.type_assert + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/modules.rst b/docs/source/api/modules.rst new file mode 100644 index 00000000..84eecc70 --- /dev/null +++ b/docs/source/api/modules.rst @@ -0,0 +1,7 @@ +maas_lib +======== + +.. toctree:: + :maxdepth: 4 + + maas_lib diff --git a/docs/source/change_log.md b/docs/source/change_log.md new file mode 100644 index 00000000..047df483 --- /dev/null +++ b/docs/source/change_log.md @@ -0,0 +1,13 @@ +## v 0.1.0 (20/05/2022) + +First internal release for pipeline inference + +* provide basic modules including fileio, logging +* config file parser +* module registry and build, which support group management +* add modules including preprocessor, model and pipeline +* image loading and nlp tokenize support in preprocessor +* add two pipeline: image-matting pipeline and text-classification pipeline +* add task constants according to PRD +* citest support +* makefile and scripts which support packaging whl, build docs, unittest diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 00000000..4a62bc1b --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,104 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +import sphinx_rtd_theme + +sys.path.insert(0, os.path.abspath('../../')) +# -- Project information ----------------------------------------------------- + +project = 'maas_lib' +copyright = '2022-2023, Alibaba PAI' +author = 'maas_lib Authors' +version_file = '../../maas_lib/version.py' + + +def get_version(): + with open(version_file, 'r') as f: + exec(compile(f.read(), version_file, 'exec')) + return locals()['__version__'] + + +# The full version, including alpha/beta/rc tags +version = get_version() +release = version + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', + 'sphinx.ext.viewcode', + 'recommonmark', + 'sphinx_markdown_tables', + 'sphinx_copybutton', +] + +autodoc_mock_imports = [ + 'matplotlib', 'pycocotools', 'terminaltables', 'mmcv.ops' +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +source_suffix = { + '.rst': 'restructuredtext', + '.md': 'markdown', +} + +# The master toctree document. +master_doc = 'index' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['build', 'Thumbs.db', '.DS_Store'] + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] +# html_css_files = ['css/readthedocs.css'] + +# -- Options for HTMLHelp output --------------------------------------------- +# Output file base name for HTML help builder. +htmlhelp_basename = 'maas_lib_doc' + +# -- Extension configuration ------------------------------------------------- +# Ignore >>> when copying code +copybutton_prompt_text = r'>>> |\.\.\. ' +copybutton_prompt_is_regexp = True + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None} + +autodoc_default_options = { + 'member-order': 'bysource', + 'special-members': '__init__', +} diff --git a/docs/source/develop.md b/docs/source/develop.md new file mode 100644 index 00000000..4d0812ae --- /dev/null +++ b/docs/source/develop.md @@ -0,0 +1,48 @@ +# Develop + +## 1. Code Style +We adopt [PEP8](https://www.python.org/dev/peps/pep-0008/) as the preferred code style. + +We use the following toolsseed isortseed isortseed isort for linting and formatting: +- [flake8](http://flake8.pycqa.org/en/latest/): linter +- [yapf](https://github.com/google/yapf): formatter +- [isort](https://github.com/timothycrosley/isort): sort imports + +Style configurations of yapf and isort can be found in [setup.cfg](../../setup.cfg). +We use [pre-commit hook](https://pre-commit.com/) that checks and formats for `flake8`, `yapf`, `seed-isort-config`, `isort`, `trailing whitespaces`, + fixes `end-of-files`, sorts `requirments.txt` automatically on every commit. + The config for a pre-commit hook is stored in [.pre-commit-config](../../.pre-commit-config.yaml). + After you clone the repository, you will need to install initialize pre-commit hook. + ```bash + pip install -r requirements/tests.txt + ``` + From the repository folder + ```bash + pre-commit install + ``` + + After this on every commit check code linters and formatter will be enforced. + + If you want to use pre-commit to check all the files, you can run + ```bash + pre-commit run --all-files + ``` + + If you only want to format and lint your code, you can run + ```bash + make linter + ``` + + ## 2. Test + ### 2.1 Unit test + ```bash + make test + ``` + + ### 2.2 Test data + TODO + + ## 3. Build pip package + ```bash + make whl + ``` diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 00000000..08e85d0b --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,43 @@ +.. maas_lib documentation file, + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +MaasLib DOCUMENTATION +======================================= + +MaasLib doc + +.. toctree:: + :maxdepth: 2 + :caption: USER GUIDE + + develop.md + +.. toctree:: + :maxdepth: 2 + :caption: Tutorials + + tutorials/index + +.. toctree:: + :maxdepth: 2 + :caption: Changelog + + change_log.md + +.. toctree:: + :maxdepth: 10 + :caption: API Doc + + api/maas_lib.preprocessors + api/maas_lib.models + api/maas_lib.pipelines + api/maas_lib.fileio + api/maas_lib.utils + + +Indices and tables +================== +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/tutorials/index.rst b/docs/source/tutorials/index.rst new file mode 100644 index 00000000..49d2cac8 --- /dev/null +++ b/docs/source/tutorials/index.rst @@ -0,0 +1,3 @@ +.. toctree:: + :maxdepth: 2 + :caption: Tutorials diff --git a/maas_lib/fileio/file.py b/maas_lib/fileio/file.py index ad890cb5..343cad9a 100644 --- a/maas_lib/fileio/file.py +++ b/maas_lib/fileio/file.py @@ -75,7 +75,7 @@ class LocalStorage(Storage): """Write data to a given ``filepath`` with 'wb' mode. Note: - ``put`` will create a directory if the directory of ``filepath`` + ``write`` will create a directory if the directory of ``filepath`` does not exist. Args: @@ -95,7 +95,7 @@ class LocalStorage(Storage): """Write data to a given ``filepath`` with 'w' mode. Note: - ``put_text`` will create a directory if the directory of + ``write_text`` will create a directory if the directory of ``filepath`` does not exist. Args: @@ -291,7 +291,7 @@ class File(object): """Write data to a given ``filepath`` with 'wb' mode. Note: - ``put`` will create a directory if the directory of ``filepath`` + ``write`` will create a directory if the directory of ``filepath`` does not exist. Args: @@ -306,7 +306,7 @@ class File(object): """Write data to a given ``filepath`` with 'w' mode. Note: - ``put_text`` will create a directory if the directory of + ``write_text`` will create a directory if the directory of ``filepath`` does not exist. Args: diff --git a/maas_lib/fileio/io.py b/maas_lib/fileio/io.py index 7f8ddd93..1b23997a 100644 --- a/maas_lib/fileio/io.py +++ b/maas_lib/fileio/io.py @@ -114,8 +114,8 @@ def dumps(obj, format, **kwargs): format (str, optional): Same as file_format :func:`load`. Examples: - >>> dumps('hello world', 'json') # disk - >>> dumps('hello world', 'yaml') # oss + >>> dumps('hello world', 'json') # json + >>> dumps('hello world', 'yaml') # yaml Returns: bool: True for success, False otherwise. diff --git a/maas_lib/models/nlp/sequence_classification_model.py b/maas_lib/models/nlp/sequence_classification_model.py index ea3076ab..db89e3bd 100644 --- a/maas_lib/models/nlp/sequence_classification_model.py +++ b/maas_lib/models/nlp/sequence_classification_model.py @@ -21,12 +21,12 @@ class SequenceClassificationModel(Model): **kwargs): # Model.__init__(self, model_dir, model_cls, first_sequence, *args, **kwargs) # Predictor.__init__(self, *args, **kwargs) - """initilize the sequence classification model from the `model_dir` path + """initilize the sequence classification model from the `model_dir` path. Args: - model_dir (str): the model path + model_dir (str): the model path. model_cls (Optional[Any], optional): model loader, if None, use the - default loader to load model weights, by default None + default loader to load model weights, by default None. """ super().__init__(model_dir, model_cls, *args, **kwargs) diff --git a/maas_lib/pipelines/builder.py b/maas_lib/pipelines/builder.py index 3a3fdaaf..12631d01 100644 --- a/maas_lib/pipelines/builder.py +++ b/maas_lib/pipelines/builder.py @@ -14,12 +14,12 @@ PIPELINES = Registry('pipelines') def build_pipeline(cfg: ConfigDict, task_name: str = None, default_args: dict = None): - """ build pipeline given model config dict + """ build pipeline given model config dict. Args: cfg (:obj:`ConfigDict`): config dict for model object. task_name (str, optional): task name, refer to - :obj:`Tasks` for more details + :obj:`Tasks` for more details. default_args (dict, optional): Default initialization arguments. """ return build_from_cfg( diff --git a/maas_lib/preprocessors/image.py b/maas_lib/preprocessors/image.py index adf70e3a..8db9f5bb 100644 --- a/maas_lib/preprocessors/image.py +++ b/maas_lib/preprocessors/image.py @@ -60,7 +60,7 @@ class LoadImage: return repr_str -def load_image(image_path_or_url: str) -> Image: +def load_image(image_path_or_url: str) -> Image.Image: """ simple interface to load an image from file or url Args: diff --git a/maas_lib/version.py b/maas_lib/version.py index b8023d8b..b794fd40 100644 --- a/maas_lib/version.py +++ b/maas_lib/version.py @@ -1 +1 @@ -__version__ = '0.0.1' +__version__ = '0.1.0' diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..35178fdf --- /dev/null +++ b/setup.py @@ -0,0 +1,183 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# !/usr/bin/env python +import os +import shutil +import subprocess +from setuptools import find_packages, setup + + +def readme(): + with open('README.md', encoding='utf-8') as f: + content = f.read() + return content + + +version_file = 'maas_lib/version.py' + + +def get_git_hash(): + + def _minimal_ext_cmd(cmd): + # construct minimal environment + env = {} + for k in ['SYSTEMROOT', 'PATH', 'HOME']: + v = os.environ.get(k) + if v is not None: + env[k] = v + # LANGUAGE is used on win32 + env['LANGUAGE'] = 'C' + env['LANG'] = 'C' + env['LC_ALL'] = 'C' + out = subprocess.Popen( + cmd, stdout=subprocess.PIPE, env=env).communicate()[0] + return out + + try: + out = _minimal_ext_cmd(['git', 'rev-parse', 'HEAD']) + sha = out.strip().decode('ascii') + except OSError: + sha = 'unknown' + + return sha + + +def get_hash(): + assert os.path.exists('.git'), '.git directory does not exist' + sha = get_git_hash()[:7] + return sha + + +def get_version(): + with open(version_file, 'r') as f: + exec(compile(f.read(), version_file, 'exec')) + return locals()['__version__'] + + +def parse_requirements(fname='requirements.txt', with_version=True): + """ + Parse the package dependencies listed in a requirements file but strips + specific versioning information. + + Args: + fname (str): path to requirements file + with_version (bool, default=False): if True include version specs + + Returns: + List[str]: list of requirements items + + CommandLine: + python -c "import setup; print(setup.parse_requirements())" + """ + import sys + from os.path import exists + import re + require_fpath = fname + + def parse_line(line): + """ + Parse information from a line in a requirements text file + """ + if line.startswith('-r '): + # Allow specifying requirements in other files + target = line.split(' ')[1] + for info in parse_require_file(target): + yield info + else: + info = {'line': line} + if line.startswith('-e '): + info['package'] = line.split('#egg=')[1] + else: + # Remove versioning from the package + pat = '(' + '|'.join(['>=', '==', '>']) + ')' + parts = re.split(pat, line, maxsplit=1) + parts = [p.strip() for p in parts] + + info['package'] = parts[0] + if len(parts) > 1: + op, rest = parts[1:] + if ';' in rest: + # Handle platform specific dependencies + # http://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-platform-specific-dependencies + version, platform_deps = map(str.strip, + rest.split(';')) + info['platform_deps'] = platform_deps + else: + version = rest # NOQA + info['version'] = (op, version) + yield info + + def parse_require_file(fpath): + with open(fpath, 'r') as f: + for line in f.readlines(): + line = line.strip() + if line.startswith('http'): + print('skip http requirements %s' % line) + continue + if line and not line.startswith('#'): + for info in parse_line(line): + yield info + + def gen_packages_items(): + if exists(require_fpath): + for info in parse_require_file(require_fpath): + parts = [info['package']] + if with_version and 'version' in info: + parts.extend(info['version']) + if not sys.version.startswith('3.4'): + # apparently package_deps are broken in 3.4 + platform_deps = info.get('platform_deps') + if platform_deps is not None: + parts.append(';' + platform_deps) + item = ''.join(parts) + yield item + + packages = list(gen_packages_items()) + return packages + + +def pack_resource(): + # pack resource such as configs and tools + root_dir = 'package/' + if os.path.isdir(root_dir): + shutil.rmtree(root_dir) + os.makedirs(root_dir) + + proj_dir = root_dir + 'maas_lib/' + shutil.copytree('./maas_lib', proj_dir) + shutil.copytree('./configs', proj_dir + 'configs') + shutil.copytree('./requirements', 'package/requirements') + shutil.copy('./requirements.txt', 'package/requirements.txt') + shutil.copy('./MANIFEST.in', 'package/MANIFEST.in') + shutil.copy('./README.md', 'package/README.md') + + +if __name__ == '__main__': + # write_version_py() + pack_resource() + os.chdir('package') + install_requires = parse_requirements('requirements.txt') + setup( + name='maas-lib', + version=get_version(), + description='', + long_description=readme(), + long_description_content_type='text/markdown', + author='Alibaba PAI team', + author_email='maas_lib@list.alibaba-inc.com', + keywords='', + url='https://github.com/alibaba/EasyCV.git', + packages=find_packages(exclude=('configs', 'tools', 'demo')), + include_package_data=True, + classifiers=[ + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + license='Apache License 2.0', + tests_require=parse_requirements('requirements/tests.txt'), + install_requires=install_requires, + zip_safe=False)