fix pandas read csv error: possible malformed input file

2026-02-24 20:19:51 +01:00 · 2024-01-02 14:21:06 +08:00
parent 27619175a9
commit bf5fa260d3
1 changed files with 2 additions and 1 deletions
--- a/modelscope/msdatasets/download/dataset_builder.py
+++ b/modelscope/msdatasets/download/dataset_builder.py
@@ -502,7 +502,8 @@ class IterableDatasetBuilder(csv.Csv):
            self.meta_csv_df = pd.read_csv(
                meta_csv_file_path,
                iterator=False,
-                delimiter=self.csv_delimiter)
+                delimiter=self.csv_delimiter,
+                lineterminator='\n')

    @staticmethod
    def trans_data_to_mapping(headers: str, texts: list, delimiter: str):