RYDE-WORK/Langchain-Chatchat
1
0
Fork 0
mirror of https://github.com/RYDE-WORK/Langchain-Chatchat.git synced 2026-02-07 15:38:27 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Dev fix csv loader (#3404)

Browse Source 
* fix: csv loader only load one column

* csv data merge with col name
This commit is contained in:
BlizBearer 2024-03-20 08:40:22 +08:00 committed by GitHub
parent d78b14721c
commit 07b7c966ed
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 19 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
document_loaders/FilteredCSVloader.py
View File

@ -59,23 +59,26 @@ class FilteredCSVLoader(CSVLoader):
docs = [] docs = []
csv_reader = csv.DictReader(csvfile, **self.csv_args) # type: ignore csv_reader = csv.DictReader(csvfile, **self.csv_args) # type: ignore
for i, row in enumerate(csv_reader): for i, row in enumerate(csv_reader):
if self.columns_to_read[0] in row: content = []
content = row[self.columns_to_read[0]] for col in self.columns_to_read:
# Extract the source if available if col in row:
source = ( content.append(f'{col}:{str(row[col])}')
row.get(self.source_column, None) else:
if self.source_column is not None raise ValueError(f"Column '{self.columns_to_read[0]}' not found in CSV file.")
else self.file_path content = '\n'.join(content)
) # Extract the source if available
metadata = {"source": source, "row": i} source = (
row.get(self.source_column, None)
if self.source_column is not None
else self.file_path
)
metadata = {"source": source, "row": i}
for col in self.metadata_columns: for col in self.metadata_columns:
if col in row: if col in row:
metadata[col] = row[col] metadata[col] = row[col]
doc = Document(page_content=content, metadata=metadata) doc = Document(page_content=content, metadata=metadata)
docs.append(doc) docs.append(doc)
else:
raise ValueError(f"Column '{self.columns_to_read[0]}' not found in CSV file.")
return docs return docs