From 9f7484bb82de6eaa8aa64d134a780a82e2ea377a Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Sun, 29 Dec 2024 13:21:01 +0800 Subject: [PATCH] wip --- modelscope/hub/check_model.py | 7 +--- modelscope/hub/create_model.py | 4 +- modelscope/utils/hf_util.py | 69 +++++++++++++++++++++++----------- 3 files changed, 51 insertions(+), 29 deletions(-) diff --git a/modelscope/hub/check_model.py b/modelscope/hub/check_model.py index 2cbfc5ef..7d094779 100644 --- a/modelscope/hub/check_model.py +++ b/modelscope/hub/check_model.py @@ -99,15 +99,12 @@ def check_local_model_is_latest( pass # ignore -def check_model_is_id(model_id: str, token=None): - if token is None: - token = os.environ.get('MODELSCOPE_API_TOKEN') +def check_model_is_id(model_id: str, token: Optional[str] = None): if model_id is None or os.path.exists(model_id): return False else: _api = HubApi() - if token is not None: - _api.login(token) + _api.try_login(token) try: _api.get_model(model_id=model_id, ) return True diff --git a/modelscope/hub/create_model.py b/modelscope/hub/create_model.py index 93ee248d..b1811acc 100644 --- a/modelscope/hub/create_model.py +++ b/modelscope/hub/create_model.py @@ -21,7 +21,7 @@ def create_model_repo(repo_id: str, Args: repo_id(str): The repo id token(str, Optional): The access token of the user - private(bool): If is a private repo + private(bool): If is a private repo, default False config_json(Dict[str, Any]): An optional config_json to fill into the configuration.json file, If None, the default content will be uploaded: ```json @@ -30,7 +30,7 @@ def create_model_repo(repo_id: str, You can manually modify this in the modelhub. """ api = HubApi() - assert repo_id is not None, 'Please enter a valid hub_model_id' + assert repo_id is not None, 'Please enter a valid repo id' api.try_login(token) visibility = ModelVisibility.PRIVATE if private else ModelVisibility.PUBLIC if '/' not in repo_id: diff --git a/modelscope/utils/hf_util.py b/modelscope/utils/hf_util.py index f1861473..859f1db5 100644 --- a/modelscope/utils/hf_util.py +++ b/modelscope/utils/hf_util.py @@ -118,15 +118,6 @@ def user_agent(invoked_by=None): return uagent -def _try_login(token: Optional[str] = None): - from modelscope.hub.api import HubApi - api = HubApi() - if token is None: - token = os.environ.get('MODELSCOPE_API_TOKEN') - if token: - api.login(token) - - def _file_exists( self, repo_id: str, @@ -141,9 +132,9 @@ def _file_exists( logger.warning( 'The passed in repo_type will not be used in modelscope. Now only model repo can be queried.' ) - _try_login(token) from modelscope.hub.api import HubApi api = HubApi() + api.try_login(token) return api.file_exists(repo_id, filename, revision=revision) @@ -171,7 +162,9 @@ def _file_download(repo_id: str, from modelscope.hub.file_download import model_file_download as file_download else: from modelscope.hub.file_download import dataset_file_download as file_download - _try_login(token) + from modelscope import HubApi + api = HubApi() + api.try_login(token) return file_download( repo_id, file_path=os.path.join(subfolder, filename) if subfolder else filename, @@ -223,11 +216,6 @@ def _patch_pretrained_class(): patch_pretrained_model_name_or_path, ori_func=PretrainedConfig.get_config_dict, ignore_file_pattern=ignore_file_pattern) - if PeftConfigHF is not None: - PeftConfigHF.from_pretrained = partial( - patch_pretrained_model_name_or_path, - ori_func=PeftConfigHF.from_pretrained, - ignore_file_pattern=ignore_file_pattern) PreTrainedModel.from_pretrained = partial( patch_pretrained_model_name_or_path, ori_func=PreTrainedModel.from_pretrained) @@ -240,6 +228,31 @@ def _patch_pretrained_class(): AutoFeatureExtractorHF.from_pretrained = partial( patch_pretrained_model_name_or_path, ori_func=AutoFeatureExtractorHF.from_pretrained) + if PeftConfigHF is not None: + PeftConfigHF.from_pretrained = partial( + patch_pretrained_model_name_or_path, + ori_func=PeftConfigHF.from_pretrained, + ignore_file_pattern=ignore_file_pattern) + + def patch_peft_model_id(cls, model, model_id, *model_args, **kwargs): + model_dir = get_model_dir(model_id, + kwargs.pop('ignore_file_pattern', None), + **kwargs) + return kwargs.pop('ori_func')(cls, model, model_dir, *model_args, **kwargs) + + if PeftModelHF is not None: + PeftModelHF.from_pretrained = partial( + patch_peft_model_id, + ori_func=PeftModelHF.from_pretrained) + PeftModelForCausalLMHF.from_pretrained = partial( + patch_peft_model_id, + ori_func=PeftModelForCausalLMHF.from_pretrained) + PeftModelForSequenceClassificationHF.from_pretrained = partial( + patch_peft_model_id, + ori_func=PeftModelForSequenceClassificationHF.from_pretrained) + PeftMixedModelHF.from_pretrained = partial( + patch_peft_model_id, + ori_func=PeftMixedModelHF.from_pretrained) def _get_peft_type(cls, model_id, **kwargs): model_dir = get_model_dir(model_id, ignore_file_pattern, **kwargs) @@ -259,21 +272,26 @@ def patch_hub(): from huggingface_hub import hf_api from huggingface_hub.hf_api import api + # Patch hf_hub_download huggingface_hub.hf_hub_download = _file_download huggingface_hub.file_download.hf_hub_download = _file_download + # Patch file_exists hf_api.file_exists = MethodType(_file_exists, api) huggingface_hub.file_exists = hf_api.file_exists huggingface_hub.hf_api.file_exists = hf_api.file_exists + # Patch whoami hf_api.whoami = MethodType(_whoami, api) huggingface_hub.whoami = hf_api.whoami huggingface_hub.hf_api.whoami = hf_api.whoami + # Patch repocard.validate from huggingface_hub import repocard repocard.RepoCard.validate = lambda *args, **kwargs: None - def create_repo(repo_id: str, + def create_repo(self, + repo_id: str, *, token: Union[str, bool, None] = None, private: bool = False, @@ -336,17 +354,24 @@ def patch_hub(): push_files_to_hub(path_or_fileobj, path_in_repo, repo_id, token, revision, commit_message, commit_description) - huggingface_hub.create_repo = create_repo - huggingface_hub.upload_folder = partial(upload_folder, api) + # Patch create_repo + from transformers.utils import hub + hf_api.create_repo = MethodType(create_repo, api) + huggingface_hub.create_repo = hf_api.create_repo + huggingface_hub.hf_api.create_repo = hf_api.create_repo + hub.create_repo = create_repo + # Patch upload_folder + hf_api.upload_folder = MethodType(upload_folder, api) + huggingface_hub.upload_folder = hf_api.upload_folder + huggingface_hub.hf_api.upload_folder = hf_api.upload_folder + + # Patch upload_file hf_api.upload_file = MethodType(upload_file, api) huggingface_hub.upload_file = hf_api.upload_file huggingface_hub.hf_api.upload_file = hf_api.upload_file repocard.upload_file = hf_api.upload_file - from transformers.utils import hub - hub.create_repo = create_repo - _patch_pretrained_class()