1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | from langchain.document_loaders import TextLoaderfrom langchain.embeddings import ModelScopeEmbeddingsfrom langchain.text_splitter import CharacterTextSplitterfrom langchain.vectorstores import Chromaimport chardet# 读取原始文档raw_documents_sanguo = TextLoader('D:\\temp\\demo.txt', encoding='utf-8').load()# 分割文档text_splitter = CharacterTextSplitter(chunk_size=100, chunk_overlap=0)documents = text_splitter.split_documents(raw_documents_sanguo)print("documents nums:", documents.__len__())print(documents)# 生成向量（embedding）model_id = "damo/nlp_corom_sentence-embedding_chinese-base"embeddings = ModelScopeEmbeddings(model_id=model_id)db = Chroma.from_documents(documents, embedding=embeddings)# 检索query = "老师说什么"docs = db.similarity_search(query, k=2)# 打印结果for doc in docs: print("===") print("metadata:", doc.metadata) print("page_content:", doc.page_content) |
