diff --git a/idrocap_ai_api/extractdata.py b/idrocap_ai_api/extractdata.py new file mode 100644 index 0000000..ab040c6 --- /dev/null +++ b/idrocap_ai_api/extractdata.py @@ -0,0 +1,84 @@ +from langchain.callbacks.manager import CallbackManager +from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler +from langchain_community.chat_models import ChatOllama +from langchain_community.document_loaders import TextLoader +from langchain.chains import RetrievalQA +from qdrant_client import QdrantClient +from langchain_community.embeddings import OllamaEmbeddings +from langchain_community.vectorstores import Qdrant +from langchain_community.chat_models import ChatOllama +from langchain_community.llms import Ollama +from langchain.callbacks.manager import CallbackManager +from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler +from langchain.prompts import PromptTemplate +import json +import datetime +def extractDataFromDoc(collection_name): + client = QdrantClient(path="./vectLlama3") + embeddings = OllamaEmbeddings(model='llama3',base_url="http://10.128.0.3:11434") + # set up the qdrant database + qdrant = Qdrant( + client=client, + collection_name=collection_name, + embeddings=embeddings + ) + +##### Ollama + +# model name can be any model you have installed with Ollama +# complete list of models available @ Ollama: https://ollama.ai/library +#llm = ChatOllama(model_name="llama3", temperature=0,base_url="http://10.128.0.3:11434" ) + llm = Ollama( + model="llama3", + base_url="http://10.128.0.3:11434", + temperature=0, + callback_manager=CallbackManager( + [StreamingStdOutCallbackHandler()] + ), + stop=["<|eot_id|>"], + ) + import json + json_schema = { + "authority-identification-code-civil-engineering-office": "restituisci il numero identificativo dell'ente pubblico, deve essere un numerico, ritorna stringa vuota se non presente", + "applicants-0-tax-code":"restituiscimi il codice fiscale del richiedente della concessione", + "applicants-0-vat-number" : "restituiscimi solo se presente, il numero di partita iva del richiedente, stringa vuota se non presente ", + "concession-decree-number" : "restituisci il numero del decreto o della pratica, restituisci solo il numero", + "release-date": "restituiscimi la data di rilascio del provvedimento", + "expiration-date": "restituiscimi la data di scadenza della concessione, eventualmente somma la durata della concessione alla data di rilascio, restituisci solo la data" + } + + + #question_1 ="restituisci solo con il json utilizzando seguente schema: " + json.dumps(json_schema, indent=2) + "le date sono in formato italiano e vanno convertite in DD-MM-YYYY, lascia vuoti i campi che non trovi" + question_1="restituisci il codice fiscale del richiedente concessione" + question_2="restituisci il numero del decreto o della pratica" + question_3="restituisci la data di inizio della concessione in formato dd/MM/yyyy. Restituisci solo la data" + question_4="restituisci la data di fine dellla concessione di utilizzo, ritorna in formato dd/MM/yyyy. Restituisci solo la data" + question_5="restituisci il numero identificativo dell'ente pubblico" + question_6="restituisci il numero di partita iva del richiedente" + # Build prompt + template = """Give a precise answer to the question based on the context without stating so. Don't be verbose. + {context} + Question:{question}""" + QA_CHAIN_PROMPT = PromptTemplate.from_template(template)# Run chain + qa_chain = RetrievalQA.from_chain_type( + llm, + retriever=qdrant.as_retriever(), + chain_type_kwargs={"prompt": QA_CHAIN_PROMPT} + ) + cf = qa_chain.invoke({"query": question_1}) + #return result["result"] + dec = qa_chain.invoke({"query": question_2}) + datar = qa_chain.invoke({"query": question_3}) + datas = qa_chain.invoke({"query": question_4}) + iduff=qa_chain.invoke({"query": question_5}) + piva=qa_chain.invoke({"query": question_6}) + response={} + response['authority-identification-code-civil-engineering-office']=iduff["result"] + response['applicants-0-tax-code']=cf["result"] + response['applicants-0-vat-number']=piva["result"] + response['concession-decree-number']=dec["result"] + response['release-date']= datetime.datetime.strptime(datar["result"].strip(), '%d/%m/%Y').strftime('%Y-%m-%d') + response['expiration-date']=datetime.datetime.strptime(datas["result"].strip(), '%d/%m/%Y').strftime('%Y-%m-%d') + return json.dumps(response) +# Check the result of the query + #print(result["result"]) diff --git a/idrocap_ai_api/main.py b/idrocap_ai_api/main.py new file mode 100644 index 0000000..6b8591a --- /dev/null +++ b/idrocap_ai_api/main.py @@ -0,0 +1,58 @@ +import time +import base64 +import sendfile +import json +from fastapi import FastAPI +from pydantic import BaseModel +from pydantic_core import from_json +from fastapi.encoders import jsonable_encoder +from fastapi.responses import JSONResponse +app = FastAPI() +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware + +app = FastAPI() + +# You can add additional URLs to this list, for example, the frontend's production domain, or other frontends. +allowed_origins = [ + "https://ai-idrocap.wgscloudlab.it", + "https://localhost", + "https://delta.webgenesys.it" +] + +app.add_middleware( + CORSMiddleware, + allow_origins=allowed_origins, + allow_credentials=True, + allow_methods=["GET", "POST", "PUT", "DELETE"], + allow_headers=["X-Requested-With", "Content-Type"], +) + +class ResponseJson(BaseModel): + authority_identification_code_civil_engineering_office: str + applicants_0_tax_code: str + applicants_0_vat_number: str + concession_decree_number: str + release_date: str + expiration_date: str + +class FileInfo(BaseModel): + type: str + documentHash: str + fileContent: str +@app.post("/extract_info/") +async def extract_info(user_data: FileInfo): + filestr = user_data.documentHash + filedata = base64.b64decode(user_data.fileContent) + filename = '%s.pdf' % filestr + with open(filename, 'wb') as f: + f.write(filedata) + resp=ResponseJson(authority_identification_code_civil_engineering_office="AA0000",applicants_0_tax_code="ABCDEF12G34H567I",applicants_0_vat_number="01234567890",concession_decree_number="12345",release_date="2024-10-23",expiration_date="2026-10-23") + jresp =jsonable_encoder(resp) + response= json.loads(sendfile.sendFile(filename,filestr)) + + return response +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=443,ssl_keyfile="./SSL/wgscloudlab.key",ssl_certfile="./SSL/wgscloudlab.crt") diff --git a/idrocap_ai_api/meta.json b/idrocap_ai_api/meta.json new file mode 100644 index 0000000..e182a76 --- /dev/null +++ b/idrocap_ai_api/meta.json @@ -0,0 +1 @@ +{"collections": {"testpdf2": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "testpdf3": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "2c7c990024bc72448117db2eb4161b7d": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "string": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "09191a028aa9a2664050247f1f50fae2": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "5bc8d65caebd8fadda6ae41e48c17a51": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "c0572767158c12300954302eff52976f": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "2614c43a78219869c0e2ce1ee6b5fea5": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "d256ee9f3d27d04a5aa58042f4abe9e6": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "c68ba602ee49340ba33a83ed97170aee": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "e65693aeee66421d598c252fb8c2d73e": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "061fb1c4423f800e19eeab394a07988a": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "e4e5b4e770164f6519563986d226dd59": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "d4a596eae81ccb17abd73a5f6823380b": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "1f1a4838648a75f31719c238b2317c4e": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "4a5ee4e037933a5346a491897624a480": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "979624fadb40ca25ffde8f90c2d82a02": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "19b98f1220476260d8afb73e4ad5e1ad": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "71936a3d0892fa9611c281f9eadd2704": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "c88962dff2f4292c9832264a27ac4cb0": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "8206869dfc40ec475264272dda261ee2": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "ef5941d5f990b97ad38deae83b18b50f": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "77a209a8e4767dc9bfb9b68a8c30cec9": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "dc9ccdeae45fa004438265ba312fa9f0": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "a9657e698ac3a7323883e095bacf4180": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "d3a8975287f2582bd809746cc01bcc0f": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "5f2c69ad1a0c16ad6ef273e19c151a0f": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "4af3bbdc6201d9fa5de100c3c652b881": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "dc2b5b41ccb1d9f257c36c4b47a760d8": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "a536e6c5bb7471c3798b25fb36a2cca0": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "0267fa9179efeac99eff832155d8566a": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "4432": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "f3091b8fbda0e85bb34ee7abd60c7381": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "690dda6963a210957359aa1d130d67f1": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "cc99517a12f228b2db0c076933fd9691": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "dd71f94b5b81b4208d1ce801c0331a35": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "ec086ddbe21b13296a6e141e51f1f091": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "gggggf": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "7f98572d22cf6042ba96fc319537f398": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "26e40eb79f30334872cc0ea243d81096": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "39a4ba21bdcfc44a0c8e456853d3fcef": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "f23000c42b3fe0be1a6b7e84f5d49f23": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "db451cf92dab8af68aa4d3c8a59f8fd0": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "cc53f35ce25fa22f765bdc86b482a098": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "352887f72361105296e923a6441ed664": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "c38493d81fe48418e302feaca9bb5f34": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "ba430bcb8eb3e20e4c1fb624aec4a71b": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "759e4d4cc7fd9f0fba77647f45018327": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "29a29a44deca0a88f382154221907aea": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "39ae76ac13482a65ccc2acfa3a80dea2": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "0443b942c03a0fc36640497370af11fb": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "8aa25c8b00f9075329cc9cddd11a9ee4": {"vectors": {"size": 4096, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}}, "aliases": {}} \ No newline at end of file diff --git a/idrocap_ai_api/readpdf.py b/idrocap_ai_api/readpdf.py new file mode 100644 index 0000000..d26111b --- /dev/null +++ b/idrocap_ai_api/readpdf.py @@ -0,0 +1,10 @@ +from pdf2image import convert_from_path +import pytesseract + +def returnTextFromFile(file): + pages = convert_from_path(file) + text="" + #text=pytesseract.image_to_string(pages[0]) + for page in pages: + text+=pytesseract.image_to_string(page) + return text diff --git a/idrocap_ai_api/sendfile.py b/idrocap_ai_api/sendfile.py new file mode 100644 index 0000000..23ae046 --- /dev/null +++ b/idrocap_ai_api/sendfile.py @@ -0,0 +1,52 @@ +import os +import readpdf +import extractdata +from qdrant_client import QdrantClient +from langchain.callbacks.manager import CallbackManager +from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler +from langchain_community.chat_models import ChatOllama +from langchain_community.document_loaders import TextLoader +from langchain.chains import RetrievalQA +from langchain.schema.document import Document +from qdrant_client.http.models import Distance, VectorParams +from langchain_community.vectorstores import Qdrant +from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter +from langchain_community.embeddings import OllamaEmbeddings +import sys + +def sendFile(filename,hash): + text= readpdf.returnTextFromFile("./"+filename) + pages=[] + + # define the text splitter + r_splitter = RecursiveCharacterTextSplitter( + chunk_size=2048, + chunk_overlap=100, + #separators=["\n\n", "\n", " ", ""] + ) + + pages = r_splitter.split_text(text) + docs = [Document(page_content=x) for x in pages] + # set up Ollama Embeddings: https://python.langchain.com/docs/integrations/text_embedding/ollama + embeddings = OllamaEmbeddings(model='llama3',base_url="http://10.128.0.3:11434") + #url="http://localhost:6333" + #client = QdrantClient(url=url) + + #client.create_collection( + # collection_name="C834_DSC_TEST", + # vectors_config=VectorParams(size=3072, distance=Distance.COSINE), + #) + # set up the qdrant database + qdrant = Qdrant.from_documents( + docs, + embeddings, + path="./vectLlama3", + collection_name=hash, + force_recreate=True, + ) + del qdrant + print ("FINE EMBEDDING") + #os.remove("./"+filename) + #print(f"File '{filename}' file rimosso.") + result = extractdata.extractDataFromDoc(hash) + return result