RYDE-WORK/Langchain-Chatchat
1
0
Fork 0
mirror of https://github.com/RYDE-WORK/Langchain-Chatchat.git synced 2026-02-07 15:38:27 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add document normalization in Chroma. (#3640)

Browse Source
This commit is contained in:
你的代码TT 2024-04-15 21:37:32 +08:00 committed by GitHub
parent 6d128f24f0
commit 2f2221ca47
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
server/knowledge_base/kb_service/chromadb_kb_service.py
View File

@ -82,9 +82,12 @@ class ChromaKBService(KBService):
def do_add_doc(self, docs: List[Document], **kwargs) -> List[Dict]: def do_add_doc(self, docs: List[Document], **kwargs) -> List[Dict]:
doc_infos = [] doc_infos = []
data = self._docs_to_embeddings(docs) embed_func = EmbeddingsFunAdapter(self.embed_model)
ids = [str(uuid.uuid1()) for _ in range(len(data["texts"]))] texts = [doc.page_content for doc in docs]
for _id, text, embedding, metadata in zip(ids, data["texts"], data["embeddings"], data["metadatas"]): metadatas = [doc.metadata for doc in docs]
embeddings = embed_func.embed_documents(texts=texts)
ids = [str(uuid.uuid1()) for _ in range(len(texts))]
for _id, text, embedding, metadata in zip(ids, texts, embeddings, metadatas):
self.collection.add(ids=_id, embeddings=embedding, metadatas=metadata, documents=text) self.collection.add(ids=_id, embeddings=embedding, metadatas=metadata, documents=text)
doc_infos.append({"id": _id, "metadata": metadata}) doc_infos.append({"id": _id, "metadata": metadata})
return doc_infos return doc_infos