Carica file su 'idrocap_ai_api'

idrocap_ai_api/extractdata.py

@@ -0,0 +1,84 @@
+from langchain.callbacks.manager import CallbackManager
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain_community.chat_models import ChatOllama
+from langchain_community.document_loaders import TextLoader
+from langchain.chains import RetrievalQA
+from qdrant_client import QdrantClient
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_community.vectorstores import Qdrant
+from langchain_community.chat_models import ChatOllama
+from langchain_community.llms import Ollama
+from langchain.callbacks.manager import CallbackManager
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain.prompts import PromptTemplate
+import json
+import datetime
+def extractDataFromDoc(collection_name):
+	client = QdrantClient(path="./vectLlama3")
+	embeddings = OllamaEmbeddings(model='llama3',base_url="http://10.128.0.3:11434")
+	# set up the qdrant database
+	qdrant = Qdrant(
+    	 client=client,
+    	 collection_name=collection_name,
+    	 embeddings=embeddings
+	)
+
+##### Ollama
+
+# model name can be any model you have installed with Ollama
+# complete list of models available @ Ollama: https://ollama.ai/library
+#llm = ChatOllama(model_name="llama3", temperature=0,base_url="http://10.128.0.3:11434" )
+	llm = Ollama(
+	model="llama3",
+	base_url="http://10.128.0.3:11434",
+	temperature=0,
+	callback_manager=CallbackManager(
+            [StreamingStdOutCallbackHandler()]
+	),
+	stop=["<|eot_id|>"],
+	)
+	import json
+	json_schema = {
+		"authority-identification-code-civil-engineering-office": "restituisci il numero identificativo dell'ente pubblico, deve essere un numerico, ritorna stringa vuota se non presente",
+        	"applicants-0-tax-code":"restituiscimi il codice fiscale del richiedente della concessione",
+		"applicants-0-vat-number" : "restituiscimi solo se presente, il numero di partita iva del richiedente, stringa vuota se non presente ",
+		"concession-decree-number" : "restituisci il numero del decreto o della pratica, restituisci solo il numero",
+		"release-date": "restituiscimi la data di rilascio del provvedimento",
+		"expiration-date": "restituiscimi la data di scadenza della concessione, eventualmente somma la durata della concessione alla data di rilascio, restituisci solo la data"
+        }
+
+
+	#question_1 ="restituisci solo con il json utilizzando seguente schema: " + json.dumps(json_schema, indent=2) + "le date sono in formato italiano e vanno convertite in DD-MM-YYYY, lascia vuoti i campi che non trovi"
+	question_1="restituisci il codice fiscale del richiedente concessione"
+	question_2="restituisci il numero del decreto o della pratica"
+	question_3="restituisci la data di inizio della concessione in formato dd/MM/yyyy. Restituisci solo la data"
+	question_4="restituisci la data di fine dellla concessione di utilizzo, ritorna in formato dd/MM/yyyy. Restituisci solo la data"
+	question_5="restituisci il numero identificativo dell'ente pubblico"
+	question_6="restituisci il numero di partita iva del richiedente"
+	# Build prompt
+	template = """Give a precise answer to the question based on the context without stating so. Don't be verbose.
+	{context}
+	Question:{question}"""
+	QA_CHAIN_PROMPT = PromptTemplate.from_template(template)# Run chain
+	qa_chain = RetrievalQA.from_chain_type(
+    	llm,
+    	retriever=qdrant.as_retriever(),
+    	chain_type_kwargs={"prompt": QA_CHAIN_PROMPT}
+	)
+	cf = qa_chain.invoke({"query": question_1})
+	#return result["result"]
+	dec = qa_chain.invoke({"query": question_2})
+	datar = qa_chain.invoke({"query": question_3})
+	datas = qa_chain.invoke({"query": question_4})
+	iduff=qa_chain.invoke({"query": question_5})
+	piva=qa_chain.invoke({"query": question_6})
+	response={}
+	response['authority-identification-code-civil-engineering-office']=iduff["result"]
+	response['applicants-0-tax-code']=cf["result"]
+	response['applicants-0-vat-number']=piva["result"]
+	response['concession-decree-number']=dec["result"]
+	response['release-date']= datetime.datetime.strptime(datar["result"].strip(), '%d/%m/%Y').strftime('%Y-%m-%d')
+	response['expiration-date']=datetime.datetime.strptime(datas["result"].strip(), '%d/%m/%Y').strftime('%Y-%m-%d') 
+	return json.dumps(response)
+# Check the result of the query
+	#print(result["result"])

idrocap_ai_api/main.py

@@ -0,0 +1,58 @@
+import time
+import base64
+import sendfile
+import json
+from fastapi import FastAPI
+from pydantic import BaseModel
+from pydantic_core import from_json
+from fastapi.encoders import jsonable_encoder
+from fastapi.responses import JSONResponse
+app = FastAPI()
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+app = FastAPI()
+
+# You can add additional URLs to this list, for example, the frontend's production domain, or other frontends.
+allowed_origins = [
+    "https://ai-idrocap.wgscloudlab.it",
+    "https://localhost",
+    "https://delta.webgenesys.it"
+]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=allowed_origins,
+    allow_credentials=True,
+    allow_methods=["GET", "POST", "PUT", "DELETE"],
+    allow_headers=["X-Requested-With", "Content-Type"],
+)
+
+class ResponseJson(BaseModel):
+    authority_identification_code_civil_engineering_office: str
+    applicants_0_tax_code: str
+    applicants_0_vat_number: str
+    concession_decree_number: str
+    release_date: str
+    expiration_date: str
+
+class FileInfo(BaseModel):
+    type: str
+    documentHash: str
+    fileContent: str
+@app.post("/extract_info/")
+async def extract_info(user_data: FileInfo):
+    filestr = user_data.documentHash
+    filedata = base64.b64decode(user_data.fileContent)
+    filename = '%s.pdf' % filestr 
+    with open(filename, 'wb') as f:
+        f.write(filedata)
+    resp=ResponseJson(authority_identification_code_civil_engineering_office="AA0000",applicants_0_tax_code="ABCDEF12G34H567I",applicants_0_vat_number="01234567890",concession_decree_number="12345",release_date="2024-10-23",expiration_date="2026-10-23")
+    jresp =jsonable_encoder(resp) 
+    response= json.loads(sendfile.sendFile(filename,filestr))
+	
+    return response
+if __name__ == "__main__":
+    import uvicorn
+ 
+    uvicorn.run(app, host="0.0.0.0", port=443,ssl_keyfile="./SSL/wgscloudlab.key",ssl_certfile="./SSL/wgscloudlab.crt")

idrocap_ai_api/readpdf.py

@@ -0,0 +1,10 @@
+from pdf2image import convert_from_path
+import pytesseract
+
+def returnTextFromFile(file):
+	pages = convert_from_path(file)
+	text=""
+	#text=pytesseract.image_to_string(pages[0])
+	for page in pages:
+   	  text+=pytesseract.image_to_string(page)
+	return text

idrocap_ai_api/sendfile.py

@@ -0,0 +1,52 @@
+import os
+import readpdf
+import extractdata
+from qdrant_client import QdrantClient
+from langchain.callbacks.manager import CallbackManager
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain_community.chat_models import ChatOllama
+from langchain_community.document_loaders import TextLoader
+from langchain.chains import RetrievalQA
+from langchain.schema.document import Document
+from qdrant_client.http.models import Distance, VectorParams
+from langchain_community.vectorstores import Qdrant
+from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter
+from langchain_community.embeddings import OllamaEmbeddings
+import sys
+
+def sendFile(filename,hash):
+	text= readpdf.returnTextFromFile("./"+filename)
+	pages=[]
+
+	# define the text splitter
+	r_splitter = RecursiveCharacterTextSplitter(
+    	  chunk_size=2048,
+          chunk_overlap=100,
+          #separators=["\n\n", "\n", " ", ""]
+	)
+
+	pages = r_splitter.split_text(text)
+	docs = [Document(page_content=x) for x in pages]
+	# set up Ollama Embeddings: https://python.langchain.com/docs/integrations/text_embedding/ollama
+	embeddings = OllamaEmbeddings(model='llama3',base_url="http://10.128.0.3:11434")
+	#url="http://localhost:6333"
+	#client = QdrantClient(url=url)
+
+	#client.create_collection(
+	#    collection_name="C834_DSC_TEST",
+	#    vectors_config=VectorParams(size=3072, distance=Distance.COSINE),
+	#)
+	# set up the qdrant database
+	qdrant = Qdrant.from_documents(
+    	 docs,
+    	 embeddings,
+    	 path="./vectLlama3",
+    	 collection_name=hash,
+    	 force_recreate=True,
+	)
+	del qdrant
+	print ("FINE EMBEDDING")
+	#os.remove("./"+filename)
+	#print(f"File '{filename}' file rimosso.")
+	result = extractdata.extractDataFromDoc(hash)
+	return result