add upload files

modelscope/hub/push_to_hub.py

@@ -3,6 +3,7 @@
 import concurrent.futures
 import os
 import shutil
+import tempfile
 from multiprocessing import Manager, Process, Value
 from pathlib import Path
 from typing import List, Optional, Union
@@ -23,6 +24,36 @@ _tasks = dict()
 _manager = None
 
 
+def push_files_to_hub(
+        path_or_fileobj: Union[str, Path],
+        path_in_repo: str,
+        repo_id: str,
+        token: Union[str, bool, None] = None,
+        revision: Optional[str] = None,
+        commit_message: Optional[str] = None,
+        commit_description: Optional[str] = None,
+):
+    if not os.path.exists(path_or_fileobj):
+        return
+
+    from modelscope import HubApi
+    api = HubApi()
+    api.login(token)
+    if not commit_message:
+        commit_message = 'Updating files'
+    if commit_description:
+        commit_message = commit_message + '\n' + commit_description
+    with tempfile.TemporaryDirectory() as temp_cache_dir:
+        from modelscope.hub.repository import Repository
+        repo = Repository(temp_cache_dir, repo_id, revision=revision)
+        sub_folder = os.path.join(temp_cache_dir, path_in_repo)
+        os.makedirs(sub_folder, exist_ok=True)
+        if os.path.isfile(path_or_fileobj):
+            shutil.copyfile(path_or_fileobj, sub_folder)
+        else:
+            shutil.copytree(path_or_fileobj, sub_folder, dirs_exist_ok=True)
+    repo.push(commit_message)
+
 def push_model_to_hub(repo_id: str,
                       folder_path: Union[str, Path],
                       path_in_repo: Optional[str] = None,

modelscope/hub/utils/utils.py

@@ -2,9 +2,11 @@
 
 import hashlib
 import os
+import shutil
+import tempfile
 from datetime import datetime
 from pathlib import Path
-from typing import List, Optional
+from typing import List, Optional, Union, BinaryIO
 
 import requests
 

modelscope/utils/hf_util.py

@@ -4,7 +4,7 @@ import tempfile
 from functools import partial
 from pathlib import Path
 from types import MethodType
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Union, BinaryIO
 from urllib.error import HTTPError
 
 from huggingface_hub.hf_api import CommitInfo, future_compatible
@@ -313,6 +313,9 @@ def patch_hub():
     huggingface_hub.whoami = hf_api.whoami
     huggingface_hub.hf_api.whoami = hf_api.whoami
 
+    from huggingface_hub.repocard import RepoCard
+    RepoCard.validate = lambda *args, **kwargs: None
+
     def create_repo(repo_id: str,
                     *,
                     token: Union[str, bool, None] = None,
@@ -359,9 +362,34 @@ def patch_hub():
             oid=None,
         )
 
+    @future_compatible
+    def upload_file(
+        self,
+        *,
+        path_or_fileobj: Union[str, Path, bytes, BinaryIO],
+        path_in_repo: str,
+        repo_id: str,
+        token: Union[str, bool, None] = None,
+        revision: Optional[str] = None,
+        commit_message: Optional[str] = None,
+        commit_description: Optional[str] = None,
+        **kwargs,
+    ):
+
+        from modelscope.hub.push_to_hub import push_files_to_hub
+        push_files_to_hub(path_or_fileobj, path_in_repo, repo_id, token, revision, commit_message, commit_description)
+
+
     huggingface_hub.create_repo = create_repo
     huggingface_hub.upload_folder = partial(upload_folder, api)
 
+    hf_api.upload_file = MethodType(upload_file, api)
+    huggingface_hub.upload_file = hf_api.upload_file
+    huggingface_hub.hf_api.upload_file = hf_api.upload_file
+
+    from transformers.utils import hub
+    hub.create_repo = create_repo
+
     _patch_pretrained_class()