first commit

Dockerfile

@@ -0,0 +1,45 @@
+FROM julia:1.10.4
+
+# Install python requirements for the project as root
+RUN apt-get update && apt-get install -y python3 python3-pip python3-venv
+
+# Create a non-root user
+RUN useradd --create-home --shell /bin/bash user
+RUN mkdir /home/user/app
+WORKDIR /home/user/app
+RUN chown -R user:user /home/
+USER user
+
+# Copy only the requirements file to leverage Docker cache
+COPY --chown=user requirements.txt /home/user/app/requirements.txt
+
+# Install pinecone and other Python dependencies as non-root user
+RUN python3 -m venv /home/user/venv && \
+    /home/user/venv/bin/pip install -r /home/user/app/requirements.txt
+
+# Copy the rest of the application code
+COPY --chown=user . /home/user/app
+
+# Copy the data to the container
+COPY --chown=user data /home/user/data
+
+# Activate the virtual environment
+RUN echo 'export PATH="/home/user/venv/bin:$PATH"' >> /home/user/.bashrc
+
+RUN mkdir -p /home/user/.julia/config && \
+    echo 'ENV["PYTHON"] = "/home/user/venv/bin/python"' >> /home/user/.julia/config/startup.jl
+
+RUN mkdir -p /home/user/.julia/config 
+#COPY startup.jl /home/user/.julia/config/startup.jl
+
+# Ensure the virtual environment is activated
+RUN /home/user/venv/bin/pip install --upgrade pip
+
+# Expose the port
+EXPOSE 8000
+EXPOSE 80
+ENV JULIA_DEPOT_PATH "/home/user/.julia"
+
+RUN julia -e 'using Pkg; Pkg.activate("."); Pkg.precompile()'
+
+ENTRYPOINT julia --project -e 'using Pkg; Pkg.instantiate(); include("server.jl")'

How to Run the OstreaCultura Server.md

@@ -0,0 +1,29 @@
+# How to Run the OstreaCultura Server Image
+
+This is a guide on how to run the OstreaCultura server image. The server image is a Docker image that contains the OstreaCultura server code and all the necessary dependencies to run the server. 
+
+
+## Prerequisites
+- Docker installed on your machine
+- Access to Internet 
+- Port 8080 is not being used by another process
+- Bash shell
+
+
+### Step 1: Pull the Docker Image
+```bash
+docker pull public.ecr.aws/a8o9b6o4/ostreacultura/api:latest
+```
+
+### Step 2: Run the Docker Image
+```bash
+docker run -p 8080:8080 -it public.ecr.aws/a8o9b6o4/ostreacultura/api:latest
+```
+
+### Step 3: Run the Server
+```bash
+cd OstreaCultura && julia --project=. server.jl
+```
+
+### Step 4: Access the Server
+Open your browser and navigate to `http://localhost:8080/docs` to access the documentation for the server.

Project.toml

@@ -0,0 +1,27 @@
+name = "OstreaCultura"
+uuid = "720c7e0a-31c4-4ba0-bde6-d3e3af4c503a"
+authors = ["stefanjwojcik and contributors"]
+version = "1.0.0-DEV"
+
+[deps]
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7"
+HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
+JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
+LocalRegistry = "89398ba2-070a-4b16-a995-9893c55d93cf"
+Oxygen = "df9a0d86-3283-4920-82dc-4555fc0d1d8b"
+Pandas = "eadc2687-ae89-51f9-a5d9-86b5a6373a9c"
+Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
+ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
+PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
+RDatasets = "ce6b1742-4840-55fa-b093-852dadbb1d8b"
+Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
+Sqids = "5846b9ac-096c-425b-b363-8d1a03210e20"
+StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[compat]
+CSV = "0.10.15"
+RDatasets = "0.7.7"

README.md

@@ -1,9 +1,10 @@
 ---
-title: Misinfo Detection App
-emoji: 🏃
-colorFrom: gray
-colorTo: green
+title: Misinformation Search
+emoji: 🌖
+colorFrom: pink
+colorTo: purple
 sdk: docker
+app_port: 8000
 pinned: false
 ---
 

ReadMe.md

@@ -0,0 +1,45 @@
+## How to run some of the code in this repository
+
+### 1. Make sure Docker is installed on your machine
+### 2. Clone the repository
+### 3. CD into the repository
+### 4. Run the following command to build the docker image
+```bash
+docker docker compose build -t oc-prototype .
+```
+### 5. Run the following command to run the docker image
+```bash
+docker compose up -d oc-prototype
+docker exec -it oc-prototype /bin/bash
+```
+
+
+## Prototype TODO's
+
+## Data
+- [X] Process all misinfo claims and generate embeddings for a library namespace 
+- [X] Upsert claims into pinecone 
+- [X] Upsert 300k into namespace
+- [ ] Update claim format to be similar to: https://www.kaggle.com/datasets/shivkumarganesh/politifact-factcheck-data/data
+
+## Functions
+- [X] Upsert vector 
+- [X] Batch upsert 
+- [X] Query against metadata
+
+- [ ] Generate working Dockerfile for project reproducibility 
+- [ ] Load data into a database 
+- [ ] Test precision/recall of embeddings
+- [ ] Generate working version of climate demo 
+
+
+Embedding pricing:
+
+1 token = approximately 0.75 words or 1k tokens = 750 words, you pay per 1000 tokens $0.0001
+Using that it can be shown that you get about 4 characters per token or 4Kb of embedding text per 1k tokens or $0.0001
+Using that as your basis you can approximate the cost of your embedding by :
+Cost in $ = Size of Data in Kilobytes * 0.000025
+
+$0.100 / 1M tokens
+
+Credentials for running google cloud queries: see ostreacultura-credentials.json

docker-compose.yaml

@@ -0,0 +1,14 @@
+version: '3.8'
+
+services:
+  ostrea_cultura:
+    container_name: my_julia_container  # Custom container name
+    build: .
+    volumes:
+      - .:/home/juliauser/OstreaCultura
+    ports:
+      - "8080:8080"
+    environment:
+      - PATH="/venv/bin:$PATH:/usr/local/julia/bin:/usr/local/bin"
+    command: /bin/bash -c "cd OstreaCultura && julia --project=. server.jl"
+    restart: unless-stopped

ostreacultura-credentials.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "ostreacultura",
+  "private_key_id": "8cf6698a8b2cf0fe9191917f1a344933995f86b3",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC4kYnebW4F0BbF\nmR5YgiqRY2nVC2oMfpRgWsjGuyeC1f1R+f8cKOG5SxWOAEjBwIKQyutO+B27uyiv\nV70vR5Z3EH+kEce+VJStnVi/2cpK1883dr9Mf7lVEoH3P5KwgYfxp0GBEZY9VO++\nBLUCaT834Al0sSGL0F8oztvV7pQ/dMDqNZ6J3OOcpg50spNuvE/Gxr5KNs3RlUiJ\nLsdJmaN7VzYZJ12u0NE2bR39m6q8BsP47O9AQfYk8UWL+oE8OY0k6EONLJEnMc+W\nYii9mmSYDvE2mP8rMpJi3dy1m8nFoz5BGiB7FGCEIaa3kqFCinss+Q1XzOKcB5PW\nIN54VoCBAgMBAAECggEABs5FdJbTNfjcjhnwtrf+B9eczgbnHy5sgqBDKdWgeSbM\nE4GR0m3NFpimyWvhqVTtVTEWVWNSyieP/xjPvtzEUJmii0ZyKua4umCRJp0XF4Sd\nShQUrVmpJNjs3LtSyZozjONrYYebfNSswaeWYRz7M5bZ4mkRrc0kmFjTkTAd9h6X\nVroxVOs2TzsELBO7LOM1TBAG0Bu5co5WlTltw8si7kEYMBJA3JQpi7BlvzMqxj3x\nlHbaMweW5fY2zpAJkWkGFxkoVg5f845nl54CsP8LWdbVH9EjLJWEqDc1GwUDCjwc\nZMjhjyeztPYzR8OhHMqu5yiknql6WtpR3+ul6khlUQKBgQDqkFj+t3quGjq5VQC1\nkQ7Vp2FpDOIisIVyzb/ETwj7G2HwW8VM630M30seEgmjO/WubcuIo3JKwt2px6Nx\nPHPp4QVR6lGUN1e7Rt1hghlCtP/RDRzErY8xHLtgzCIcKfQDQFNJLh4ti0IHqsQJ\n7nYtXpyRsvRPXIvttXLgA7sMxQKBgQDJb4sMhPkZLZ3MlhNc24oeL4btXh0D9PPV\nGbwcjIKZA/DyI+w7TXvW1LvwaWhr0ZopDkCJ5qtnwpIN91pq19CJzpH23iqrtszt\ng+nOekQ51yH27WSbLKcKeotC1PmpR/WhlFBJIihtKmAtpBGue73PNegmCcT/Lkrn\n4x54ioYYjQKBgHqy63rCq6/rFJDiAC76JFJq7i+vh5Mx13zIV+nulfoUxtSejMTB\n4nYbUAX41YXH1apa0L83EJA/AwjO35ZaUyAij0/cTLRFhuT2GtPo5CHa4H6QZ14S\nk/bS2sm1vpoQ8A3LVkXPlAWq9O1hzI6NaNAuz6P/goEJHkFIS0EXcn8pAoGADODi\nxPQjcOanthLKPh+THwbu23Iu7LfWdwNkMXcSSnIkD3cYfVog1sVXFrHGiU4nSwSs\nP6TNxJ599AKidz5BCO56Cox9sejIBU9vvVG31vLYKw9WZU1fiJ35FVbEd3wwShpP\nX72dVA1nhMN+lec3ZW4O34+0VpRBGb49jntcw2kCgYEA5VMAHskd23+mgsemIAkX\nOZS8dAjlJ2X1lFhOn9tCK3QI1NnQpyIxGK+iYjQjre/aFvT+5XpqUmvZ/deDax8b\nd1sCR5hF/NcDuqkcmBNkkZrt+cdwqyd/LHkPEtMVVcLHjoXh7r/VfwY38IbXpcUB\n30YPr+31bqumyYzssUc98Cw=\n-----END PRIVATE KEY-----\n",
+  "client_email": "[email protected]",
+  "client_id": "117283783846337502690",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/ocservice%40ostreacultura.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}

requirements.txt

@@ -0,0 +1,6 @@
+pandas
+numpy
+sentence-transformers
+pinecone[grpc]
+pinecone_text
+sqids

server.jl

@@ -0,0 +1,52 @@
+using Oxygen
+using HTTP
+import OstreaCultura as OC
+
+# Load the fasttext embeddings and the fasttext model
+const (fc_embed, fc) = OC.load_fasttext_embeddings("data/filtered_fact_check_latest_embed.csv")
+const (nar_embed, nar) = OC.load_fasttext_embeddings("data/expansive_claims_library_expanded_embed.csv")
+
+@get "/greet" function(req::HTTP.Request)
+    return "hello world!"
+end
+
+## Send a query to the test-index index in the test-namespace namespace
+#@get "/query" function(req::HTTP.Request, querytext::String, indexname::String, namespace::String, top_k::Int64=5, include_values::Bool=false)
+#    OC.query(querytext, indexname, namespace; top_k=top_k, include_values=include_values)
+#end
+
+## Send a query to look for matches within an organic dataset 
+#@get "/queryclaims" function(req::HTTP.Request, claim::String, counterclaim::String, indexname::String, namespace::String, top_k::Int64=5000, include_values::Bool=false)
+#    OC.query_claims(claim, counterclaim, indexname, namespace; top_k=top_k, include_values=include_values)
+#end
+
+## Classify a claim within the misinformation library 
+#@get "/classify" function(req::HTTP.Request, claim::String, counterclaim::String, indexname::String, namespace::String, top_k::Int64=10, include_values::Bool=false)
+#    OC.classify_claim(claim, counterclaim, indexname, namespace; top_k=top_k, include_values=include_values)
+#end
+
+# Model is really 'namespace' in the OC library
+@get "/search" function(req::HTTP.Request, claim::String, model::String="narratives", top_k::Int64=5)
+    # remove % signs from claim, replace with percent 
+    claim = replace(claim, "%" => "percent")
+    OC.search(claim, "oc-hybrid-library-index", model; top_k=top_k, include_values=false)
+end
+
+@get "/fastfactsearch" function(req::HTTP.Request, claim::String, model::String, top_k::Int64=5)
+    # remove % signs from claim, replace with percent 
+    claim = replace(claim, "%" => "percent")
+    if model == "narratives"
+        json(OC.fast_topk(nar_embed, nar, claim, top_k))
+    elseif model == "factchecks"
+        json(OC.fast_topk(fc_embed, fc, claim, top_k))
+    else
+        "Model not found"
+    end
+end
+
+#@get "/searchplot" function(req::HTTP.Request, claim::String, model::String, top_k::Int64=5, include_values::Bool=false)
+#    OC.searchplot(claim, "oc-hybrid-library-index", model; top_k=top_k, include_values=include_values)
+#end
+
+# start the web server
+serve(host="0.0.0.0", port=8000)

startup.jl

@@ -0,0 +1,19 @@
+ENV["PYTHON"]="/venv/bin/python"
+ENV["JULIA_NUM_THREADS"] = 8
+ENV["JULIA_PKG_DEVDIR"] = "/home/workspace"
+
+ENV["OPENAI_API_KEY"] = "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"
+ENV["PINECONE_API_KEY"] = "5faec954-a6c5-4af5-a577-89dbd2e4e5b0"
+
+using Revise 
+
+function restart()
+    startup = """
+        Base.ACTIVE_PROJECT[]=$(repr(Base.ACTIVE_PROJECT[]))
+        Base.HOME_PROJECT[]=$(repr(Base.HOME_PROJECT[]))
+        cd($(repr(pwd())))
+        """
+    cmd = `$(Base.julia_cmd()) -ie $startup`
+    atexit(()->run(cmd))
+    exit(0)
+end