[to #42461396] add git-lfs support and mv test data to git-lfs

.gitattributes

@@ -0,0 +1,3 @@
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.mp4 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -104,7 +104,6 @@ venv.bak/
 # mypy
 .mypy_cache/
 
-data
 .vscode
 .idea
 

data/test/images/image1.jpg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78094cc48fbcfd9b6d321fe13619ecc72b65e006fc1b4c4458409ade9979486d
+size 129862

data/test/images/image_matting.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af83a94899a6d23339c3ecc5c4c58c57c835af57b531a2f4c50461184f820141
+size 603621

docs/source/develop.md

@@ -91,6 +91,55 @@ make tests
 
 4. Daily regression tests will run all cases at 0 am each day using master branch.
 
+### 2.3 Test data storage
+
+As we need a lot of data for testing, including images, videos, models. We use git lfs
+to store those large files.
+
+1. install git-lfs
+for mac
+```bash
+brew install git-lfs
+git lfs install
+```
+
+for centos, please download rpm from git-lfs github release [website](https://github.com/git-lfs/git-lfs/releases/tag/v3.2.0)
+```bash
+wget http://101374-public.oss-cn-hangzhou-zmf.aliyuncs.com/git-lfs-3.2.0-1.el7.x86_64.rpm
+sudo rpm -ivh git-lfs-3.2.0-1.el7.x86_64.rpm
+git lfs install
+```
+
+for ubuntu
+```bash
+curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
+sudo apt-get install git-lfs
+git lfs install
+```
+
+2. track your data type using git lfs, for example, to track png files
+```bash
+git lfs track "*.png"
+```
+
+3. add your test files to `data/test/` folder, you can make directories if you need.
+```bash
+git add data/test/test.png
+```
+
+4. commit your test data to remote branch
+```bash
+git commit -m "xxx"
+```
+
+To pull data from remote repo, just as the same way you pull git files.
+```bash
+git pull origin branch_name
+```
+
+
+
+
 ## Code Review
 
 1. Run following command to create an aone CR, replace `TARGET_BRANCH` and `CR_NAME` with the one you want.

tests/pipelines/test_base.py

@@ -80,8 +80,7 @@ class CustomPipelineTest(unittest.TestCase):
         pipe2 = pipeline(dummy_task)
         self.assertTrue(type(pipe) is type(pipe2))
 
-        img_url = 'http://pai-vision-data-hz.oss-cn-zhangjiakou.' \
-                  'aliyuncs.com/data/test/images/image1.jpg'
+        img_url = 'data/test/images/image1.jpg'
         output = pipe(img_url)
         self.assertEqual(output['filename'], img_url)
         self.assertEqual(output['output_png'].shape, (318, 512, 3))

tests/pipelines/test_image_captioning.py

@@ -27,9 +27,7 @@ class ImageCaptionTest(unittest.TestCase):
             img_captioning = pipeline(
                 Tasks.image_captioning, model=ofile.name, bpe_dir=bpe_dir)
 
-            result = img_captioning(
-                'http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'
-            )
+            result = img_captioning('data/test/images/image_matting.png')
             print(result['caption'])
 
 

tests/pipelines/test_image_matting.py

@@ -34,16 +34,12 @@ class ImageMattingTest(unittest.TestCase):
                 ofile.write(File.read(model_path))
             img_matting = pipeline(Tasks.image_matting, model=tmp_dir)
 
-            result = img_matting(
-                'http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'
-            )
+            result = img_matting('data/test/images/image_matting.png')
             cv2.imwrite('result.png', result['output_png'])
 
     @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
     def test_run_with_dataset(self):
-        input_location = [
-            'http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'
-        ]
+        input_location = ['data/test/images/image_matting.png']
         # alternatively:
         # input_location = '/dir/to/images'
 
@@ -58,9 +54,7 @@ class ImageMattingTest(unittest.TestCase):
     def test_run_modelhub(self):
         img_matting = pipeline(Tasks.image_matting, model=self.model_id)
 
-        result = img_matting(
-            'http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'
-        )
+        result = img_matting('data/test/images/image_matting.png')
         cv2.imwrite('result.png', result['output_png'])
         print(f'Output written to {osp.abspath("result.png")}')
 
@@ -68,9 +62,7 @@ class ImageMattingTest(unittest.TestCase):
     def test_run_modelhub_default_model(self):
         img_matting = pipeline(Tasks.image_matting)
 
-        result = img_matting(
-            'http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'
-        )
+        result = img_matting('data/test/images/image_matting.png')
         cv2.imwrite('result.png', result['output_png'])
         print(f'Output written to {osp.abspath("result.png")}')
 

tests/preprocessors/test_image.py

@@ -11,9 +11,7 @@ from modelscope.utils.logger import get_logger
 class ImagePreprocessorTest(unittest.TestCase):
 
     def test_load(self):
-        img = load_image(
-            'http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'
-        )
+        img = load_image('data/test/images/image_matting.png')
         self.assertTrue(isinstance(img, PIL.Image.Image))
         self.assertEqual(img.size, (948, 533))