Non puoi selezionare più di 25 argomenti
Gli argomenti devono iniziare con una lettera o un numero, possono includere trattini ('-') e possono essere lunghi fino a 35 caratteri.
52 righe
1.7 KiB
52 righe
1.7 KiB
import os |
|
import readpdf |
|
import extractdata |
|
from qdrant_client import QdrantClient |
|
from langchain.callbacks.manager import CallbackManager |
|
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler |
|
from langchain_community.chat_models import ChatOllama |
|
from langchain_community.document_loaders import TextLoader |
|
from langchain.chains import RetrievalQA |
|
from langchain.schema.document import Document |
|
from qdrant_client.http.models import Distance, VectorParams |
|
from langchain_community.vectorstores import Qdrant |
|
from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter |
|
from langchain_community.embeddings import OllamaEmbeddings |
|
import sys |
|
|
|
def sendFile(filename,hash): |
|
text= readpdf.returnTextFromFile("./"+filename) |
|
pages=[] |
|
|
|
# define the text splitter |
|
r_splitter = RecursiveCharacterTextSplitter( |
|
chunk_size=2048, |
|
chunk_overlap=100, |
|
#separators=["\n\n", "\n", " ", ""] |
|
) |
|
|
|
pages = r_splitter.split_text(text) |
|
docs = [Document(page_content=x) for x in pages] |
|
# set up Ollama Embeddings: https://python.langchain.com/docs/integrations/text_embedding/ollama |
|
embeddings = OllamaEmbeddings(model='llama3',base_url="http://10.128.0.3:11434") |
|
#url="http://localhost:6333" |
|
#client = QdrantClient(url=url) |
|
|
|
#client.create_collection( |
|
# collection_name="C834_DSC_TEST", |
|
# vectors_config=VectorParams(size=3072, distance=Distance.COSINE), |
|
#) |
|
# set up the qdrant database |
|
qdrant = Qdrant.from_documents( |
|
docs, |
|
embeddings, |
|
path="./vectLlama3", |
|
collection_name=hash, |
|
force_recreate=True, |
|
) |
|
del qdrant |
|
print ("FINE EMBEDDING") |
|
#os.remove("./"+filename) |
|
#print(f"File '{filename}' file rimosso.") |
|
result = extractdata.extractDataFromDoc(hash) |
|
return result
|
|
|