diff --git a/app.py b/app.py index 211729e..d4f61d3 100644 --- a/app.py +++ b/app.py @@ -4,34 +4,41 @@ from flask import Flask, request, jsonify, render_template, send_from_directory from main import consult_simba_oracle -app = Flask(__name__, static_folder="raggr-frontend/dist/static", template_folder="raggr-frontend/dist") +app = Flask( + __name__, + static_folder="raggr-frontend/dist/static", + template_folder="raggr-frontend/dist", +) # Serve React static files -@app.route('/static/') +@app.route("/static/") def static_files(filename): return send_from_directory(app.static_folder, filename) + # Serve the React app for all routes (catch-all) -@app.route('/', defaults={'path': ''}) -@app.route('/') +@app.route("/", defaults={"path": ""}) +@app.route("/") def serve_react_app(path): if path and os.path.exists(os.path.join(app.template_folder, path)): return send_from_directory(app.template_folder, path) - return render_template('index.html') + return render_template("index.html") + @app.route("/api/query", methods=["POST"]) def query(): - data = request.get_json() + data = request.get_json() query = data.get("query") return jsonify({"response": consult_simba_oracle(query)}) + @app.route("/api/ingest", methods=["POST"]) def webhook(): data = request.get_json() print(data) return jsonify({"status": "received"}) + if __name__ == "__main__": app.run(host="0.0.0.0", port=8080, debug=True) - diff --git a/image_process.py b/image_process.py new file mode 100644 index 0000000..7cef2e6 --- /dev/null +++ b/image_process.py @@ -0,0 +1,81 @@ +from ollama import Client +import argparse +import os +import logging +from PIL import Image, ExifTags +from pillow_heif import register_heif_opener +from pydantic import BaseModel + +from dotenv import load_dotenv + +load_dotenv() + +register_heif_opener() + +logging.basicConfig(level=logging.INFO) + + +parser = argparse.ArgumentParser( + prog="SimbaImageProcessor", + description="What the program does", + epilog="Text at the bottom of help", +) + +parser.add_argument("filepath") + +client = Client(host=os.getenv("OLLAMA_HOST", "http://localhost:11434")) + +class SimbaImageDescription(BaseModel): + image_date: str + description: str + +def describe_simba_image(input): + logging.info("Opening image of Simba ...") + if "heic" in input.lower() or "heif" in input.lower(): + new_filepath = input.split(".")[0] + ".jpg" + img = Image.open(input) + img.save(new_filepath, 'JPEG') + logging.info("Extracting EXIF...") + exif = { + ExifTags.TAGS[k]: v for k, v in img.getexif().items() if k in ExifTags.TAGS + } + img = Image.open(new_filepath) + input=new_filepath + else: + img = Image.open(input) + + logging.info("Extracting EXIF...") + exif = { + ExifTags.TAGS[k]: v for k, v in img.getexif().items() if k in ExifTags.TAGS + } + + if "MakerNote" in exif: + exif.pop("MakerNote") + + logging.info(exif) + + prompt = f"Simba is an orange cat belonging to Ryan Chen. In 2025, they lived in New York. In 2024, they lived in California. Analyze the following image and tell me what Simba seems to be doing. Be extremely descriptive about Simba, things in the background, and the setting of the image. I will also include the EXIF data of the image, please use it to help you determine information about Simba. EXIF: {exif}. Put the notes in the description field and the date in the image_date field." + + logging.info("Sending info to Ollama ...") + response = client.chat( + model="gemma3:4b", + messages=[ + { + "role": "system", + "content": "you are a very shrewd and descriptive note taker. all of your responses will be formatted like notes in bullet points. be very descriptive. do not leave a single thing out.", + }, + {"role": "user", "content": prompt, "images": [input]}, + ], + format=SimbaImageDescription.model_json_schema() + ) + + result = SimbaImageDescription.model_validate_json(response["message"]["content"]) + + return result + + +if __name__ == "__main__": + args = parser.parse_args() + if args.filepath: + logging.info + describe_simba_image(input=args.filepath) diff --git a/index_immich.py b/index_immich.py new file mode 100644 index 0000000..ad12e2f --- /dev/null +++ b/index_immich.py @@ -0,0 +1,98 @@ +import httpx +import os +from pathlib import Path +import logging +import tempfile + +from image_process import describe_simba_image +from request import PaperlessNGXService + +logging.basicConfig(level=logging.INFO) + + +from dotenv import load_dotenv + +load_dotenv() + +# Configuration from environment variables +IMMICH_URL = os.getenv("IMMICH_URL", "http://localhost:2283") +API_KEY = os.getenv("IMMICH_API_KEY") +PERSON_NAME = os.getenv("PERSON_NAME", "Simba") # Name of the tagged person/pet +DOWNLOAD_DIR = os.getenv("DOWNLOAD_DIR", "./simba_photos") + +# Set up headers +headers = {"x-api-key": API_KEY, "Content-Type": "application/json"} + + +if __name__ == "__main__": + ppngx = PaperlessNGXService() + people_url = f"{IMMICH_URL}/api/search/person?name=Simba" + people = httpx.get(people_url, headers=headers).json() + + simba_id = people[0]["id"] + + ids = {} + + asset_search = f"{IMMICH_URL}/api/search/smart" + request_body = {"query": "orange cat"} + results = httpx.post(asset_search, headers=headers, json=request_body) + + assets = results.json()["assets"] + for asset in assets["items"]: + if asset["type"] == "IMAGE": + ids[asset["id"]] = asset.get("originalFileName") + nextPage = assets.get("nextPage") + + # while nextPage != None: + # logging.info(f"next page: {nextPage}") + # request_body["page"] = nextPage + # results = httpx.post(asset_search, headers=headers, json=request_body) + # assets = results.json()["assets"] + + # for asset in assets["items"]: + # if asset["type"] == "IMAGE": + # ids.add(asset['id']) + + # nextPage = assets.get("nextPage") + + asset_search = f"{IMMICH_URL}/api/search/smart" + request_body = {"query": "simba"} + results = httpx.post(asset_search, headers=headers, json=request_body) + print(results.json()["assets"]["total"]) + for asset in results.json()["assets"]["items"]: + if asset["type"] == "IMAGE": + ids[asset["id"]] = asset.get("originalFileName") + + immich_asset_id = list(ids.keys())[1] + immich_filename = ids.get(immich_asset_id) + response = httpx.get( + f"{IMMICH_URL}/api/assets/{immich_asset_id}/original", headers=headers + ) + + path = os.path.join("/Users/ryanchen/Programs/raggr", immich_filename) + file = open(path, "wb+") + for chunk in response.iter_bytes(chunk_size=8192): + file.write(chunk) + + logging.info("Processing image ...") + description = describe_simba_image(path) + + image_description = description.description + image_date = description.image_date + + description_filepath = os.path.join("/Users/ryanchen/Programs/raggr", f"SIMBA_DESCRIBE_001.txt") + file = open(description_filepath, "w+") + file.write(image_description) + file.close() + + file = open(description_filepath, 'rb') + + ppngx.upload_description(description_filepath=description_filepath, file=file, title="SIMBA_DESCRIBE_001.txt", exif_date=image_date) + + + file.close() + + + + logging.info("Processing complete. Deleting file.") + os.remove(file.name) diff --git a/main.py b/main.py index 1d13463..4e58b33 100644 --- a/main.py +++ b/main.py @@ -33,14 +33,13 @@ parser.add_argument("query", type=str, help="questions about simba's health") parser.add_argument( "--reindex", action="store_true", help="re-index the simba documents" ) -parser.add_argument( - "--index", help="index a file" -) +parser.add_argument("--index", help="index a file") ppngx = PaperlessNGXService() openai_client = OpenAI() + def index_using_pdf_llm(): files = ppngx.get_data() for file in files: @@ -79,14 +78,15 @@ def chunk_data(docs: list[dict[str, Union[str, Any]]], collection): for index, text in enumerate(texts): print(docs[index]["original_file_name"]) metadata = { - "created_date": date_to_epoch(docs[index]["created_date"]), - "filename": docs[index]["original_file_name"] + "created_date": date_to_epoch(docs[index]["created_date"]), + "filename": docs[index]["original_file_name"], } chunker.chunk_document( document=text, metadata=metadata, ) + def chunk_text(texts: list[str], collection): chunker = Chunker(collection) @@ -97,9 +97,11 @@ def chunk_text(texts: list[str], collection): metadata=metadata, ) + def consult_oracle(input: str, collection): print(input) import time + start_time = time.time() # Ask @@ -122,7 +124,7 @@ def consult_oracle(input: str, collection): results = collection.query( query_texts=[input], query_embeddings=embeddings, - #where=metadata_filter, + # where=metadata_filter, ) print(results) query_end = time.time() @@ -132,15 +134,21 @@ def consult_oracle(input: str, collection): print("Starting LLM generation") llm_start = time.time() # output = ollama_client.generate( - # model="gemma3n:e4b", - # prompt=f"You are a helpful assistant that understandings veterinary terms. Using the following data, help answer the user's query by providing as many details as possible. Using this data: {results}. Respond to this prompt: {input}", + # model="gemma3n:e4b", + # prompt=f"You are a helpful assistant that understandings veterinary terms. Using the following data, help answer the user's query by providing as many details as possible. Using this data: {results}. Respond to this prompt: {input}", # ) response = openai_client.chat.completions.create( model="gpt-4o-mini", messages=[ - {"role": "system", "content": "You are a helpful assistant that understands veterinary terms."}, - {"role": "user", "content": f"Using the following data, help answer the user's query by providing as many details as possible. Using this data: {results}. Respond to this prompt: {input}"} - ] + { + "role": "system", + "content": "You are a helpful assistant that understands veterinary terms.", + }, + { + "role": "user", + "content": f"Using the following data, help answer the user's query by providing as many details as possible. Using this data: {results}. Respond to this prompt: {input}", + }, + ], ) llm_end = time.time() print(f"LLM generation took {llm_end - llm_start:.2f} seconds") @@ -181,7 +189,6 @@ if __name__ == "__main__": print("Done chunking documents") # index_using_pdf_llm() - if args.index: with open(args.index) as file: extension = args.index.split(".")[-1] @@ -196,11 +203,11 @@ if __name__ == "__main__": if args.query: print("Consulting oracle ...") - print(consult_oracle( - input=args.query, - collection=simba_docs, - )) + print( + consult_oracle( + input=args.query, + collection=simba_docs, + ) + ) else: print("please provide a query") - - diff --git a/pyproject.toml b/pyproject.toml index 92084cb..afa81db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,4 +14,6 @@ dependencies = [ "pydantic>=2.11.9", "pillow>=10.0.0", "pymupdf>=1.24.0", + "black>=25.9.0", + "pillow-heif>=1.1.1", ] diff --git a/query.py b/query.py index 1e29be5..927bbee 100644 --- a/query.py +++ b/query.py @@ -33,9 +33,11 @@ class GeneratedQuery(BaseModel): fields: list[str] extracted_metadata_fields: str + class Time(BaseModel): time: int + PROMPT = """ You are an information specialist that processes user queries. The current year is 2025. The user queries are all about a cat, Simba, and its records. The types of records are listed below. Using the query, extract the @@ -114,16 +116,16 @@ class QueryGenerator: query = json.loads(response.output_parsed.extracted_metadata_fields) # response: ChatResponse = ollama_client.chat( - # model="gemma3n:e4b", - # messages=[ - # {"role": "system", "content": PROMPT}, - # {"role": "user", "content": input}, - # ], - # format=GeneratedQuery.model_json_schema(), + # model="gemma3n:e4b", + # messages=[ + # {"role": "system", "content": PROMPT}, + # {"role": "user", "content": input}, + # ], + # format=GeneratedQuery.model_json_schema(), # ) # query = json.loads( - # json.loads(response["message"]["content"])["extracted_metadata_fields"] + # json.loads(response["message"]["content"])["extracted_metadata_fields"] # ) date_key = list(query["created_date"].keys())[0] query["created_date"][date_key] = self.date_to_epoch( diff --git a/raggr-frontend/src/App.tsx b/raggr-frontend/src/App.tsx index a2b8eb3..f335937 100644 --- a/raggr-frontend/src/App.tsx +++ b/raggr-frontend/src/App.tsx @@ -33,50 +33,56 @@ const App = () => { setQuery(event.target.value); }; return ( -
-
-
-

ask simba!

-
-
-