mirror of
https://github.com/modelscope/modelscope.git
synced 2026-09-02 12:09:36 +02:00
976bdf4ebc
1. Add Hubs.virgo in MsDataset.load() 2. Add VirgoDownloader pipeline 3. Add auth for virgo hub. 4. Add other utils. 5. cherry-pick from https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12352198?file=8c317278a40b19fe76825fb749e768fdbfd6a711 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12351633
37 lines
783 B
Python
37 lines
783 B
Python
# Copyright (c) Alibaba, Inc. and its affiliates.
|
|
|
|
from urllib.parse import urlparse
|
|
|
|
import pandas as pd
|
|
|
|
from modelscope.utils.logger import get_logger
|
|
|
|
logger = get_logger()
|
|
|
|
|
|
def valid_url(url) -> bool:
|
|
try:
|
|
result = urlparse(url)
|
|
return all([result.scheme, result.netloc])
|
|
except ValueError as e:
|
|
logger.warning(e)
|
|
return False
|
|
|
|
|
|
def fetch_csv_with_url(csv_url: str) -> pd.DataFrame:
|
|
"""Fetch the csv content from url.
|
|
|
|
Args:
|
|
csv_url (str): The input url of csv data.
|
|
|
|
Returns:
|
|
A pandas DataFrame object which contains the csv content.
|
|
"""
|
|
try:
|
|
df = pd.read_csv(csv_url)
|
|
except Exception as e:
|
|
logger.error(f'Failed to fetch csv from url: {csv_url}')
|
|
raise e
|
|
|
|
return df
|