From bf5fa260d3635eaccac86b29ee217521e3d66328 Mon Sep 17 00:00:00 2001
From: "xingjun.wang" <xingjun.wxj@alibaba-inc.com>
Date: Tue, 2 Jan 2024 14:21:06 +0800
Subject: [PATCH] fix pandas read csv error: `possible malformed input file`

---
 modelscope/msdatasets/download/dataset_builder.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modelscope/msdatasets/download/dataset_builder.py b/modelscope/msdatasets/download/dataset_builder.py
index 33640315..5daf84a7 100644
--- a/modelscope/msdatasets/download/dataset_builder.py
+++ b/modelscope/msdatasets/download/dataset_builder.py
@@ -502,7 +502,8 @@ class IterableDatasetBuilder(csv.Csv):
             self.meta_csv_df = pd.read_csv(
                 meta_csv_file_path,
                 iterator=False,
-                delimiter=self.csv_delimiter)
+                delimiter=self.csv_delimiter,
+                lineterminator='\n')
 
     @staticmethod
     def trans_data_to_mapping(headers: str, texts: list, delimiter: str):