mirror of
https://github.com/modelscope/modelscope.git
synced 2025-12-16 16:27:45 +01:00
2df80c0c40
Co-authored-by: Yingda Chen <yingda.chen@alibaba-inc.com>
(cherry picked from commit 1f88654aa1)
85 lines
2.9 KiB
Python
85 lines
2.9 KiB
Python
# Copyright (c) Alibaba, Inc. and its affiliates.
|
|
import os
|
|
import tempfile
|
|
import unittest
|
|
|
|
from requests import HTTPError
|
|
|
|
from modelscope.fileio.file import File, HTTPStorage, LocalStorage
|
|
from modelscope.hub.utils.utils import convert_patterns
|
|
|
|
|
|
class FileTest(unittest.TestCase):
|
|
|
|
def test_pattern_conversion(self):
|
|
self._assert_patterns(None, None)
|
|
self._assert_patterns('*.h5', ['*.h5'])
|
|
self._assert_patterns('*.h5 ', ['*.h5'])
|
|
self._assert_patterns('*.h5, *flax_model.msgpack',
|
|
['*.h5', '*flax_model.msgpack'])
|
|
self._assert_patterns(['*.h5, *flax_model.msgpack'],
|
|
['*.h5', '*flax_model.msgpack'])
|
|
self._assert_patterns(['*.h5 ', '*flax_model.msgpack'],
|
|
['*.h5', '*flax_model.msgpack'])
|
|
|
|
def _assert_patterns(self, raw_input, expected_output):
|
|
output = convert_patterns(raw_input)
|
|
self.assertEqual(expected_output, output)
|
|
|
|
def test_local_storage(self):
|
|
storage = LocalStorage()
|
|
temp_name = tempfile.gettempdir() + '/' + next(
|
|
tempfile._get_candidate_names())
|
|
binary_content = b'12345'
|
|
storage.write(binary_content, temp_name)
|
|
self.assertEqual(binary_content, storage.read(temp_name))
|
|
|
|
content = '12345'
|
|
storage.write_text(content, temp_name)
|
|
self.assertEqual(content, storage.read_text(temp_name))
|
|
|
|
os.remove(temp_name)
|
|
|
|
def test_http_storage(self):
|
|
storage = HTTPStorage()
|
|
url = 'https://modelscope.oss-cn-beijing.aliyuncs.com/test/texts/data.txt'
|
|
content = 'this is test data'
|
|
self.assertEqual(content.encode('utf8'), storage.read(url))
|
|
self.assertEqual(content, storage.read_text(url))
|
|
|
|
with storage.as_local_path(url) as local_file:
|
|
with open(local_file, 'r') as infile:
|
|
self.assertEqual(content, infile.read())
|
|
|
|
with self.assertRaises(NotImplementedError):
|
|
storage.write('dfad', url)
|
|
|
|
with self.assertRaises(HTTPError):
|
|
storage.read(url + 'df')
|
|
|
|
def test_file(self):
|
|
url = 'https://modelscope.oss-cn-beijing.aliyuncs.com/test/texts/data.txt'
|
|
content = 'this is test data'
|
|
self.assertEqual(content.encode('utf8'), File.read(url))
|
|
|
|
with File.as_local_path(url) as local_file:
|
|
with open(local_file, 'r') as infile:
|
|
self.assertEqual(content, infile.read())
|
|
|
|
with self.assertRaises(NotImplementedError):
|
|
File.write('dfad', url)
|
|
|
|
with self.assertRaises(HTTPError):
|
|
File.read(url + 'df')
|
|
|
|
temp_name = tempfile.gettempdir() + '/' + next(
|
|
tempfile._get_candidate_names())
|
|
binary_content = b'12345'
|
|
File.write(binary_content, temp_name)
|
|
self.assertEqual(binary_content, File.read(temp_name))
|
|
os.remove(temp_name)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|