From 12e474f9d6c07bc82e5749e8d751c39617b4fd5a Mon Sep 17 00:00:00 2001 From: liuyhwangyh Date: Fri, 28 Jun 2024 16:19:58 +0800 Subject: [PATCH] Refactor install dependencies (#889) * refactor install dependencies, default only depencies of hub and datasets * move pandas import to function * update hub deps * test * remove generate ast file * remove gast dependency * replace gast with ast * add dependency version * remove lap for compile error * fix comments issue * add install ollama --------- Co-authored-by: mulin.lyh --- docker/Dockerfile.ubuntu | 2 + modelscope/__init__.py | 4 -- modelscope/cli/server.py | 7 +-- modelscope/fileio/__init__.py | 22 +++++++++- modelscope/fileio/format/json.py | 7 +-- modelscope/fileio/format/jsonplus.py | 2 +- modelscope/hub/api.py | 3 +- modelscope/server/__init__.py | 17 ++++++++ modelscope/server/api/routers/router.py | 1 - modelscope/server/api_server.py | 58 +++++++++++++++---------- modelscope/utils/__init__.py | 1 - modelscope/utils/ast_utils.py | 30 ++++++++++--- modelscope/utils/automodel_utils.py | 3 +- modelscope/utils/config.py | 5 ++- modelscope/utils/constant.py | 2 + modelscope/utils/import_utils.py | 3 +- modelscope/utils/logger.py | 6 +-- modelscope/utils/plugins.py | 3 +- requirements.txt | 2 +- requirements/cv.txt | 5 +-- requirements/framework.txt | 1 - requirements/hub.txt | 3 ++ requirements/{svr.txt => server.txt} | 1 - setup.py | 9 +++- 24 files changed, 133 insertions(+), 64 deletions(-) create mode 100644 requirements/hub.txt rename requirements/{svr.txt => server.txt} (76%) diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index 6d4b4c0f..437ee9e2 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -9,6 +9,8 @@ ARG CUDA_VERSION=cu121 # install jupyter plugin RUN mkdir -p /root/.local/share/jupyter/labextensions/ && \ cp -r /tmp/resources/jupyter_plugins/* /root/.local/share/jupyter/labextensions/ +# install ollama +RUN curl -fsSL https://ollama.com/install.sh | sh COPY docker/scripts/modelscope_env_init.sh /usr/local/bin/ms_env_init.sh # python3.8 pip install git+https://github.com/jin-s13/xtcocoapi.git@v1.13 diff --git a/modelscope/__init__.py b/modelscope/__init__.py index 01630ab5..8f2fbbee 100644 --- a/modelscope/__init__.py +++ b/modelscope/__init__.py @@ -96,10 +96,6 @@ else: 'AutoModelForTokenClassification', 'AutoImageProcessor', 'BatchFeature' ] - else: - print( - 'transformer is not installed, please install it if you want to use related modules' - ) import sys diff --git a/modelscope/cli/server.py b/modelscope/cli/server.py index 0f8b8132..17d6ca4d 100644 --- a/modelscope/cli/server.py +++ b/modelscope/cli/server.py @@ -4,10 +4,8 @@ import os from argparse import ArgumentParser from string import Template -import uvicorn - from modelscope.cli.base import CLICommand -from modelscope.server.api_server import add_server_args, get_app +from modelscope.server.api_server import add_server_args, run_server from modelscope.utils.logger import get_logger logger = get_logger(log_level=logging.WARNING) @@ -37,5 +35,4 @@ class ServerCMD(CLICommand): parser.set_defaults(func=subparser_func) def execute(self): - app = get_app(self.args) - uvicorn.run(app, host=self.args.host, port=self.args.port) + run_server(self.args) diff --git a/modelscope/fileio/__init__.py b/modelscope/fileio/__init__.py index 385cd02c..986affb2 100644 --- a/modelscope/fileio/__init__.py +++ b/modelscope/fileio/__init__.py @@ -1,4 +1,22 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -from .file import File, LocalStorage -from .io import dump, dumps, load +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .file import File, LocalStorage + from .io import dump, dumps, load +else: + _import_structure = { + 'io': ['dump', 'dumps', 'load'], + 'file': ['File', 'LocalStorage'] + } + import sys + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/modelscope/fileio/format/json.py b/modelscope/fileio/format/json.py index 660e342a..cb26e58d 100644 --- a/modelscope/fileio/format/json.py +++ b/modelscope/fileio/format/json.py @@ -1,11 +1,9 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -import numpy as np - -from . import jsonplus from .base import FormatHandler def set_default(obj): + import numpy as np """Set default json values for non-serializable values. It helps convert ``set``, ``range`` and ``np.ndarray`` data types to list. @@ -25,10 +23,13 @@ class JsonHandler(FormatHandler): """Use jsonplus, serialization of Python types to JSON that "just works".""" def load(self, file): + from . import jsonplus return jsonplus.loads(file.read()) def dump(self, obj, file, **kwargs): + from . import jsonplus file.write(self.dumps(obj, **kwargs)) def dumps(self, obj, **kwargs): + from . import jsonplus return jsonplus.dumps(obj, **kwargs) diff --git a/modelscope/fileio/format/jsonplus.py b/modelscope/fileio/format/jsonplus.py index af59caeb..48a4d512 100644 --- a/modelscope/fileio/format/jsonplus.py +++ b/modelscope/fileio/format/jsonplus.py @@ -9,7 +9,7 @@ import simplejson as json import threading import uuid from collections import namedtuple -from datetime import date, datetime, time, timedelta +from datetime import timedelta from dateutil.parser import parse as parse_datetime from decimal import Decimal from fractions import Fraction diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 717264dd..fe61e31f 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -18,7 +18,6 @@ from typing import Dict, List, Optional, Tuple, Union from urllib.parse import urlencode import json -import pandas as pd import requests from requests import Session from requests.adapters import HTTPAdapter, Retry @@ -820,6 +819,8 @@ class HubApi: """ import hashlib from tqdm import tqdm + import pandas as pd + out_path = os.path.join(out_path, hashlib.md5(url.encode(encoding='UTF-8')).hexdigest()) if mode == DownloadMode.FORCE_REDOWNLOAD and os.path.exists(out_path): os.remove(out_path) diff --git a/modelscope/server/__init__.py b/modelscope/server/__init__.py index e69de29b..587aa148 100644 --- a/modelscope/server/__init__.py +++ b/modelscope/server/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .api_server import run_server, add_server_args +else: + _import_structure = {'api_server': ['run_server', 'add_server_arg']} + import sys + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/modelscope/server/api/routers/router.py b/modelscope/server/api/routers/router.py index df1a1868..3f35edf1 100644 --- a/modelscope/server/api/routers/router.py +++ b/modelscope/server/api/routers/router.py @@ -1,5 +1,4 @@ from fastapi import APIRouter -from starlette.routing import Route, WebSocketRoute from modelscope.server.api.routers import health, model_router diff --git a/modelscope/server/api_server.py b/modelscope/server/api_server.py index 99d20275..594f53af 100644 --- a/modelscope/server/api_server.py +++ b/modelscope/server/api_server.py @@ -1,28 +1,7 @@ import argparse -import uvicorn -from fastapi import FastAPI -from modelscope.server.api.routers.router import api_router -from modelscope.server.core.event_handlers import (start_app_handler, - stop_app_handler) - - -def get_app(args) -> FastAPI: - app = FastAPI( - title='modelscope_server', - version='0.1', - debug=True, - swagger_ui_parameters={'tryItOutEnabled': True}) - app.state.args = args - app.include_router(api_router) - - app.add_event_handler('startup', start_app_handler(app)) - app.add_event_handler('shutdown', stop_app_handler(app)) - return app - - -def add_server_args(parser): +def add_server_args(parser: argparse.ArgumentParser): parser.add_argument( '--model_id', required=True, type=str, help='The target model id') parser.add_argument( @@ -37,7 +16,42 @@ def add_server_args(parser): help='Use LLMPipeline first for llm models.') +def run_server(args): + try: + import uvicorn + app = get_app(args) + uvicorn.run(app, host=args.host, port=args.port) + except ModuleNotFoundError as e: + print(e) + print( + 'To execute the server command, first ' + 'install the domain dependencies with: ' + 'pip install modelscope[DOMAIN] -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html ' + 'the "DOMAIN" include [cv|nlp|audio|multi-modal|science] ' + 'and then install server dependencies with: pip install modelscope[server]' + ) + + +def get_app(args): + from fastapi import FastAPI + from modelscope.server.api.routers.router import api_router + from modelscope.server.core.event_handlers import (start_app_handler, + stop_app_handler) + app = FastAPI( + title='modelscope_server', + version='0.1', + debug=True, + swagger_ui_parameters={'tryItOutEnabled': True}) + app.state.args = args + app.include_router(api_router) + + app.add_event_handler('startup', start_app_handler(app)) + app.add_event_handler('shutdown', stop_app_handler(app)) + return app + + if __name__ == '__main__': + import uvicorn parser = argparse.ArgumentParser('modelscope_server') add_server_args(parser) args = parser.parse_args() diff --git a/modelscope/utils/__init__.py b/modelscope/utils/__init__.py index 7486e137..e69de29b 100644 --- a/modelscope/utils/__init__.py +++ b/modelscope/utils/__init__.py @@ -1 +0,0 @@ -from .hub import create_model_if_not_exist, read_config diff --git a/modelscope/utils/ast_utils.py b/modelscope/utils/ast_utils.py index 6b9ba463..72627e2d 100644 --- a/modelscope/utils/ast_utils.py +++ b/modelscope/utils/ast_utils.py @@ -11,10 +11,8 @@ from functools import reduce from pathlib import Path from typing import Union -import gast import json -from modelscope.fileio.file import LocalStorage # do not delete from modelscope.metainfo import (CustomDatasets, Heads, Hooks, LR_Schedulers, Metrics, Models, Optimizers, Pipelines, @@ -25,7 +23,6 @@ from modelscope.utils.logger import get_logger from modelscope.utils.registry import default_group logger = get_logger(log_level=logging.WARNING) -storage = LocalStorage() p = Path(__file__) # get the path of package 'modelscope' @@ -362,8 +359,7 @@ class AstScanning(object): with open(file, 'r', encoding='utf8') as code: data = code.readlines() data = ''.join(data) - - node = gast.parse(data) + node = ast.parse(data) output = self.scan_import(node, show_offsets=False) output[DECORATOR_KEY] = self.parse_decorators(output[DECORATOR_KEY]) output[EXPRESS_KEY] = self.parse_decorators(output[EXPRESS_KEY]) @@ -575,6 +571,25 @@ class FilesAstScanning(object): file_scanner = FilesAstScanning() +def ensure_write(obj: bytes, filepath: Union[str, Path]) -> None: + """Write data to a given ``filepath`` with 'wb' mode. + + Note: + ``write`` will create a directory if the directory of ``filepath`` + does not exist. + + Args: + obj (bytes): Data to be written. + filepath (str or Path): Path to write data. + """ + dirname = os.path.dirname(filepath) + if dirname and not os.path.exists(dirname): + os.makedirs(dirname, exist_ok=True) + + with open(filepath, 'wb') as f: + f.write(obj) + + def _save_index(index, file_path, file_list=None, with_template=False): # convert tuple key to str key index[INDEX_KEY] = {str(k): v for k, v in index[INDEX_KEY].items()} @@ -587,7 +602,7 @@ def _save_index(index, file_path, file_list=None, with_template=False): if with_template: json_index = json_index.replace(MODELSCOPE_PATH.as_posix(), TEMPLATE_PATH) - storage.write(json_index.encode(), file_path) + ensure_write(json_index.encode(), file_path) index[INDEX_KEY] = { ast.literal_eval(k): v for k, v in index[INDEX_KEY].items() @@ -595,7 +610,8 @@ def _save_index(index, file_path, file_list=None, with_template=False): def _load_index(file_path, with_template=False): - bytes_index = storage.read(file_path) + with open(file_path, 'rb') as f: + bytes_index = f.read() if with_template: bytes_index = bytes_index.decode().replace(TEMPLATE_PATH, MODELSCOPE_PATH.as_posix()) diff --git a/modelscope/utils/automodel_utils.py b/modelscope/utils/automodel_utils.py index f96046ff..5cefdafe 100644 --- a/modelscope/utils/automodel_utils.py +++ b/modelscope/utils/automodel_utils.py @@ -3,8 +3,6 @@ import os from types import MethodType from typing import Any, Optional -from packaging import version - from modelscope.metainfo import Tasks from modelscope.utils.ast_utils import INDEX_KEY from modelscope.utils.import_utils import (LazyImportModule, @@ -42,6 +40,7 @@ def fix_transformers_upgrade(): # from 4.35.0, transformers changes its arguments of _set_gradient_checkpointing import transformers from transformers import PreTrainedModel + from packaging import version if version.parse(transformers.__version__) >= version.parse('4.35.0') \ and not hasattr(PreTrainedModel, 'post_init_origin'): PreTrainedModel.post_init_origin = PreTrainedModel.post_init diff --git a/modelscope/utils/config.py b/modelscope/utils/config.py index 85cb8b77..099dbf11 100644 --- a/modelscope/utils/config.py +++ b/modelscope/utils/config.py @@ -16,10 +16,8 @@ from typing import Dict, Union import addict import json -from yapf.yapflib.yapf_api import FormatCode from modelscope.utils.constant import ConfigFields, ModelFile -from modelscope.utils.import_utils import import_modules_from_file from modelscope.utils.logger import get_logger logger = get_logger() @@ -101,6 +99,8 @@ class Config: shutil.copyfile(filename, tmp_cfg_file.name) if filename.endswith('.py'): + # import as needed. + from modelscope.utils.import_utils import import_modules_from_file module_nanme, mod = import_modules_from_file( osp.join(tmp_cfg_dir, tmp_cfg_name)) cfg_dict = {} @@ -282,6 +282,7 @@ class Config: based_on_style='pep8', blank_line_before_nested_class_or_def=True, split_before_expression_after_opening_paren=True) + from yapf.yapflib.yapf_api import FormatCode text, _ = FormatCode(text, style_config=yapf_style, verify=True) return text diff --git a/modelscope/utils/constant.py b/modelscope/utils/constant.py index 08850e5e..11be17fb 100644 --- a/modelscope/utils/constant.py +++ b/modelscope/utils/constant.py @@ -5,11 +5,13 @@ import enum class Fields(object): """ Names for different application fields """ + framework = 'framework' cv = 'cv' nlp = 'nlp' audio = 'audio' multi_modal = 'multi-modal' science = 'science' + server = 'server' class CVTasks(object): diff --git a/modelscope/utils/import_utils.py b/modelscope/utils/import_utils.py index f1a9ad70..8c897ddb 100644 --- a/modelscope/utils/import_utils.py +++ b/modelscope/utils/import_utils.py @@ -14,8 +14,6 @@ from pathlib import Path from types import ModuleType from typing import Any -from packaging import version - from modelscope.utils.ast_utils import (INDEX_KEY, MODULE_KEY, REQUIREMENT_KEY, load_index) from modelscope.utils.error import * # noqa @@ -193,6 +191,7 @@ if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VA pass _tf_available = _tf_version is not None if _tf_available: + from packaging import version if version.parse(_tf_version) < version.parse('2'): pass else: diff --git a/modelscope/utils/logger.py b/modelscope/utils/logger.py index be1c24d3..8d4174e8 100644 --- a/modelscope/utils/logger.py +++ b/modelscope/utils/logger.py @@ -1,6 +1,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -import importlib +import importlib.util as iutil import logging from typing import Optional @@ -41,7 +41,7 @@ def get_logger(log_file: Optional[str] = None, # at the ERROR level. torch_dist = False is_worker0 = True - if importlib.util.find_spec('torch') is not None: + if iutil.find_spec('torch') is not None: from modelscope.utils.torch_utils import is_dist, is_master torch_dist = is_dist() is_worker0 = is_master() @@ -78,7 +78,7 @@ def add_file_handler_if_needed(logger, log_file, file_mode, log_level): if isinstance(handler, logging.FileHandler): return - if importlib.util.find_spec('torch') is not None: + if iutil.find_spec('torch') is not None: from modelscope.utils.torch_utils import is_master is_worker0 = is_master() else: diff --git a/modelscope/utils/plugins.py b/modelscope/utils/plugins.py index e0731c8c..d9441cf1 100644 --- a/modelscope/utils/plugins.py +++ b/modelscope/utils/plugins.py @@ -17,11 +17,11 @@ from typing import Any, Iterable, List, Optional, Set, Union import json import pkg_resources +from modelscope import snapshot_download from modelscope.fileio.file import LocalStorage from modelscope.utils.ast_utils import FilesAstScanning from modelscope.utils.constant import DEFAULT_MODEL_REVISION from modelscope.utils.file_utils import get_modelscope_cache_dir -from modelscope.utils.hub import read_config, snapshot_download from modelscope.utils.logger import get_logger logger = get_logger() @@ -1140,6 +1140,7 @@ class EnvsManager(object): cache_dir = os.getenv('MODELSCOPE_CACHE', cache_dir) self.env_dir = os.path.join(cache_dir, EnvsManager.name, model_id) model_dir = snapshot_download(model_id, revision=model_revision) + from modelscope.utils.hub import read_config cfg = read_config(model_dir) self.plugins = cfg.get('plugins', []) self.allow_remote = cfg.get('allow_remote', False) diff --git a/requirements.txt b/requirements.txt index 0832e6ab..daa2b59c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ --r requirements/framework.txt +-r requirements/hub.txt diff --git a/requirements/cv.txt b/requirements/cv.txt index c8edb672..5935fd91 100644 --- a/requirements/cv.txt +++ b/requirements/cv.txt @@ -23,7 +23,6 @@ imageio>=2.9.0 imageio-ffmpeg>=0.4.2 imgaug>=0.4.0 kornia>=0.5.0 -lap lmdb lpips ml_collections @@ -52,8 +51,8 @@ PyMCubes pytorch-lightning regex # <0.20.0 for compatible python3.7 python3.8 -scikit-image>=0.19.3,<0.20.0 -scikit-learn>=0.20.1 +scikit-image +scikit-learn shapely shotdetect_scenedetect_lgss>=0.0.4 smplx diff --git a/requirements/framework.txt b/requirements/framework.txt index d091c13f..d4428ae9 100644 --- a/requirements/framework.txt +++ b/requirements/framework.txt @@ -3,7 +3,6 @@ attrs datasets>=2.16.0,<2.19.0 einops filelock>=3.3.0 -gast>=0.2.2 huggingface_hub numpy oss2 diff --git a/requirements/hub.txt b/requirements/hub.txt new file mode 100644 index 00000000..ce8c7562 --- /dev/null +++ b/requirements/hub.txt @@ -0,0 +1,3 @@ +requests>=2.25 +tqdm>=4.64.0 +urllib3>=1.26 diff --git a/requirements/svr.txt b/requirements/server.txt similarity index 76% rename from requirements/svr.txt rename to requirements/server.txt index ea439c66..ffe131c0 100644 --- a/requirements/svr.txt +++ b/requirements/server.txt @@ -1,4 +1,3 @@ fastapi -requests sse-starlette uvicorn diff --git a/setup.py b/setup.py index dbac6e77..d722ee41 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,6 @@ import shutil import subprocess from setuptools import find_packages, setup -from modelscope.utils.ast_utils import generate_ast_template from modelscope.utils.constant import Fields @@ -171,6 +170,7 @@ def pack_resource(): if __name__ == '__main__': # write_version_py() + from modelscope.utils.ast_utils import generate_ast_template generate_ast_template() pack_resource() os.chdir('package') @@ -192,6 +192,13 @@ if __name__ == '__main__': filed_name = f'audio_{subfiled}' extra_requires[filed_name], _ = parse_requirements( f'requirements/audio/{filed_name}.txt') + framework_requires = extra_requires['framework'] + # add framework dependencies to every field + for field, requires in extra_requires.items(): + if field not in [ + 'server', 'framework' + ]: # server need install model's field dependencies before. + extra_requires[field] = framework_requires + extra_requires[field] extra_requires['all'] = all_requires setup(