populate db, db optimize

app.py

@@ -3,182 +3,210 @@ import json
 import re
 import sqlite3
 import logging
+from collections import defaultdict
 from util import process_json_files
 from gematria import calculate_gematria
 from deep_translator import GoogleTranslator, exceptions
 
-logging.basicConfig(level=logging.INFO, format='%(message)s')
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Global variables for database connection and translator
+conn = None
+translator = None
 
 def flatten_text(text):
-    """Helper function to flatten nested lists into a single list."""
-    if isinstance(text, list):
-        return " ".join(flatten_text(item) if isinstance(item, list) else item for item in text)
-    return text
+  """Helper function to flatten nested lists into a single list."""
+  if isinstance(text, list):
+    return " ".join(flatten_text(item) if isinstance(item, list) else item for item in text)
+  return text
 
 def initialize_database():
-    conn = sqlite3.connect('gematria.db')
-    c = conn.cursor()
+  """Initializes the SQLite database."""
+  global conn
+  conn = sqlite3.connect('gematria.db')
+  c = conn.cursor()
+  c.execute('''
+    CREATE TABLE IF NOT EXISTS results (
+      gematria_sum INTEGER,
+      words TEXT UNIQUE,
+      translation TEXT,
+      occurrences TEXT,
+      PRIMARY KEY (words)
+    )
+  ''')
+  c.execute('''
+    CREATE TABLE IF NOT EXISTS processed_books (
+      book INTEGER PRIMARY KEY,
+      max_phrase_length INTEGER
+    )
+  ''')
+  conn.commit()
+  logging.info("Database initialized.")
+
+def initialize_translator():
+  """Initializes the Google Translator."""
+  global translator
+  translator = GoogleTranslator(source='iw', target='en')
+  logging.info("Translator initialized.")
+
+def insert_phrase_to_db(gematria_sum, phrase_candidate, translation, occurrence):
+  """Inserts a phrase and its Gematria value into the database."""
+  global conn
+  c = conn.cursor()
+  try:
     c.execute('''
-        CREATE TABLE IF NOT EXISTS results (
-            gematria_sum INTEGER,
-            words TEXT,
-            translation TEXT,
-            book INTEGER,
-            title TEXT,
-            chapter INTEGER,
-            verse INTEGER,
-            UNIQUE(gematria_sum, words, book, title, chapter, verse)
-        )
-    ''')
+      INSERT INTO results (gematria_sum, words, translation, occurrences)
+      VALUES (?, ?, ?, ?)
+    ''', (gematria_sum, phrase_candidate, translation, occurrence))
+    conn.commit()
+    logging.debug(f"Inserted phrase: {phrase_candidate} (Gematria: {gematria_sum})")
+  except sqlite3.IntegrityError:
+    logging.debug(f"Phrase already exists: {phrase_candidate} (Gematria: {gematria_sum})")
+    c.execute('''
+      SELECT occurrences FROM results WHERE words = ?
+    ''', (phrase_candidate,))
+    existing_occurrences = c.fetchone()[0]
+    updated_occurrences = existing_occurrences + ';' + occurrence
+    c.execute('''
+      UPDATE results
+      SET occurrences = ?
+      WHERE words = ?
+    ''', (updated_occurrences, phrase_candidate))
     conn.commit()
-    conn.close()
 
-def insert_phrase_to_db(c, gematria_sum, phrase_candidate, translation, book_id, title, chapter_id, verse_id):
+def populate_database(tanach_texts, max_phrase_length=3):
+  """Populates the database with phrases from the Tanach and their Gematria values."""
+  global conn
+  logging.info("Populating database...")
+  c = conn.cursor()
+  for book_id, text in enumerate(tanach_texts):
+    c.execute('''SELECT max_phrase_length FROM processed_books WHERE book = ?''', (book_id + 1,))
+    result = c.fetchone()
+    if result and result[0] >= max_phrase_length:
+      logging.info(f"Skipping book {book_id+1}: Already processed with max_phrase_length {result[0]}")
+      continue
+
+    logging.info(f"Processing book {book_id+1} with max_phrase_length {max_phrase_length}")
+    if 'text' not in text or not isinstance(text['text'], list):
+      logging.warning(f"Skipping book {book_id} due to missing or invalid 'text' field.")
+      continue
+    title = text.get('title', 'Unknown')
+    chapters = text['text']
+    for chapter_id, chapter in enumerate(chapters):
+      if not isinstance(chapter, list):
+        logging.warning(f"Skipping chapter {chapter_id} in book {title} due to invalid format.")
+        continue
+      for verse_id, verse in enumerate(chapter):
+        verse_text = flatten_text(verse)
+        verse_text = re.sub(r"[^\u05D0-\u05EA ]+", "", verse_text)
+        verse_text = re.sub(r" +", " ", verse_text)
+        words = verse_text.split()
+        for length in range(1, max_phrase_length + 1):
+          for start in range(len(words) - length + 1):
+            phrase_candidate = " ".join(words[start:start + length])
+            gematria_sum = calculate_gematria(phrase_candidate.replace(" ", ""))
+            occurrence = f"{book_id+1}:{title}:{chapter_id+1}:{verse_id+1}"
+            insert_phrase_to_db(gematria_sum, phrase_candidate, None, occurrence)  # No translation initially
     try:
-        logging.info(f"Inserting: {gematria_sum}, {phrase_candidate}, {translation}, {book_id}, {title}, {chapter_id}, {verse_id}")
-        c.execute('''
-            INSERT INTO results (gematria_sum, words, translation, book, title, chapter, verse)
-            VALUES (?, ?, ?, ?, ?, ?, ?)
-        ''', (gematria_sum, phrase_candidate, translation, book_id, title, chapter_id, verse_id))
+      c.execute('''INSERT INTO processed_books (book, max_phrase_length) VALUES (?, ?)''', (book_id + 1, max_phrase_length))
     except sqlite3.IntegrityError:
-        logging.info(f"Entry already exists: {gematria_sum}, {phrase_candidate}, {book_id}, {title}, {chapter_id}, {verse_id}")
-
-def populate_database(tanach_texts, max_phrase_length=1):
-    conn = sqlite3.connect('gematria.db')
-    c = conn.cursor()
-    for book_id, text in enumerate(tanach_texts):
-        if 'text' not in text or not isinstance(text['text'], list):
-            logging.warning(f"Skipping book {book_id} due to missing or invalid 'text' field.")
-            continue
-        title = text.get('title', 'Unknown')
-        chapters = text['text']
-        for chapter_id, chapter in enumerate(chapters):
-            if not isinstance(chapter, list):
-                logging.warning(f"Skipping chapter {chapter_id} in book {title} due to invalid format.")
-                continue
-            for verse_id, verse in enumerate(chapter):
-                verse_text = flatten_text(verse) 
-                verse_text = re.sub(r"[^\u05D0-\u05EA ]+", "", verse_text)
-                verse_text = re.sub(r" +", " ", verse_text)
-                words = verse_text.split()
-                max_length = min(max_phrase_length, len(words))
-                for length in range(1, max_length + 1):
-                    for start in range(len(words) - length + 1):
-                        phrase_candidate = " ".join(words[start:start + length])
-                        gematria_sum = calculate_gematria(phrase_candidate.replace(" ", ""))
-                        insert_phrase_to_db(c, gematria_sum, phrase_candidate, None, book_id + 1, title, chapter_id + 1, verse_id + 1) # No translation initially
+      c.execute('''UPDATE processed_books SET max_phrase_length = ? WHERE book = ?''', (max_phrase_length, book_id + 1))
     conn.commit()
-    conn.close()
-
-def get_translation_from_db(c, phrase, book, chapter, verse):
+  logging.info("Database population complete.")
+
+def get_translation(phrase):
+  """Retrieves or generates the English translation of a Hebrew phrase."""
+  global translator, conn
+  c = conn.cursor()
+  c.execute('''
+    SELECT translation FROM results
+    WHERE words = ?
+  ''', (phrase,))
+  result = c.fetchone()
+  if result and result[0]:
+    return result[0]
+  else:
+    translation = translate_and_store(phrase)
     c.execute('''
-        SELECT translation FROM results 
-        WHERE words = ? AND book = ? AND chapter = ? AND verse = ?
-    ''', (phrase, book, chapter, verse))
-    result = c.fetchone()
-    return result[0] if result else None
-
-def translate_and_store(conn, phrase, book, chapter, verse):
-    translator = GoogleTranslator(source='iw', target='en')  # Explicitly set source to Hebrew
-    c = conn.cursor()
-    try:
-        translation = translator.translate(phrase)
-        logging.info(f"Translated phrase: {translation}")
-        c.execute('''
-            UPDATE results 
-            SET translation = ?
-            WHERE words = ? AND book = ? AND chapter = ? AND verse = ?
-        ''', (translation, phrase, book, chapter, verse))
-        conn.commit()
-        return translation
-    except (exceptions.TranslationNotFound, exceptions.NotValidPayload, 
-            exceptions.ServerException, exceptions.RequestError) as e:
-        logging.error(f"Error translating phrase '{phrase}': {e}")
-        return "[Translation Error]"
+      UPDATE results
+      SET translation = ?
+      WHERE words = ?
+    ''', (translation, phrase))
+    conn.commit()
+    return translation
+
+
+def translate_and_store(phrase):
+  """Translates a phrase using Google Translate."""
+  global translator
+  try:
+    translation = translator.translate(phrase)
+    logging.debug(f"Translated phrase: {translation}")
+    return translation
+  except (exceptions.TranslationNotFound, exceptions.NotValidPayload,
+          exceptions.ServerException, exceptions.RequestError) as e:
+    logging.error(f"Error translating phrase '{phrase}': {e}")
+    return "[Translation Error]"
+
+def search_gematria_in_db(gematria_sum):
+  """Searches the database for phrases with a given Gematria value."""
+  global conn
+  c = conn.cursor()
+  c.execute('''
+    SELECT words, occurrences FROM results WHERE gematria_sum = ?
+  ''', (gematria_sum,))
+  results = c.fetchall()
+  logging.debug(f"Found {len(results)} matching phrases for Gematria: {gematria_sum}")
+  return results
 
 def gematria_search_interface(phrase):
-    debug_output = []
-
-    def debug_callback(message):
-        debug_output.append(message)
-        logging.info(message)
-
-    if not phrase.strip():
-        return "Please enter a phrase.", "\n".join(debug_output)
-
-    phrase_gematria = calculate_gematria(phrase.replace(" ", ""))
-    debug_callback(f"Debug: Gematria of the search phrase '{phrase}' is {phrase_gematria}")
-
-    conn = sqlite3.connect('gematria.db')
-    c = conn.cursor()
-    matching_phrases = search_gematria_in_db(c, phrase_gematria)
-
-    if not matching_phrases:
-        conn.close()
-        return "No matching phrases found.", "\n".join(debug_output)
-
-    # Sort matching phrases by book, chapter, and verse
-    matching_phrases.sort(key=lambda x: (x[1], x[3], x[4]))
-
-    result = "Matching phrases:\n"
-    for match in matching_phrases:
-        if len(match) != 6:  # Adjusted length for added translation
-            debug_callback(f"Error: Expected tuple of length 6, but got {len(match)}: {match}")
-            continue
-        words, book, title, chapter, verse, translation = match
-        if not translation: # Check if translation exists
-            translation = translate_and_store(conn, words, book, chapter, verse)
-        result += f"Book: {title} ({book})\nChapter: {chapter}, Verse: {verse}\nPhrase: {words}\nTranslation: {translation}\n\n"
-
-    conn.close()
-    return result, "\n".join(debug_output)
-
-def search_gematria_in_db(c, gematria_sum):
-    c.execute('''
-        SELECT words, book, title, chapter, verse, translation FROM results WHERE gematria_sum = ?
-    ''', (gematria_sum,))
-    results = c.fetchall()
-    logging.info(f"Search results: {results}")
-    return results
-
-def run_test():
-    debug_output = []
-    #test_phrase = "אחר ואתבנימין ואני"
-    #expected_gematria = 1495
-
-    def debug_callback(message):
-        debug_output.append(message)
-        logging.info(message)
-
-    # Load the test JSON contents for 01.json
-    #test_texts_00 = process_json_files(0, 0)
-    #test_texts_01 = process_json_files(1, 1)
-    #populate_database(test_texts_00, max_phrase_length=22)  # Populate the database from book 0 with phrases up to 22 words
-    #populate_database(test_texts_01, max_phrase_length=3)  # Populate the database from book 1 with phrases up to 3 words
-    #conn = sqlite3.connect('gematria.db')
-    #c = conn.cursor()
-    #matching_phrases_01 = search_gematria_in_db(c, expected_gematria)
-    #conn.close()
-    #assert matching_phrases_01[0][0] == test_phrase, f"Found phrase does not match: {matching_phrases_01[0][0]}"
-    #print("Test successful: The phrase was correctly found and the gematria matches in 01.json.")
-    #print("\n".join(debug_output))
-
-    test_texts = process_json_files(1, 39)
-    populate_database(test_texts, max_phrase_length=1)
-    #populate_database(test_texts, max_phrase_length=2)
-    #populate_database(test_texts, max_phrase_length=3)
-
-iface = gr.Interface(
+  """The main function for the Gradio interface."""
+  if not phrase.strip():
+    return "Please enter a phrase."
+
+  # Datenbankverbindung innerhalb der Funktion erstellen
+  global conn
+  conn = sqlite3.connect('gematria.db')
+  c = conn.cursor()
+
+  phrase_gematria = calculate_gematria(phrase.replace(" ", ""))
+  logging.info(f"Searching for phrases with Gematria: {phrase_gematria}")
+
+  matching_phrases = search_gematria_in_db(phrase_gematria)
+  if not matching_phrases:
+    return "No matching phrases found."
+
+  # Format results for display
+  results = []
+  for words, occurrences in matching_phrases:
+    translation = get_translation(words)
+    for occurrence in occurrences.split(';'):
+      book, title, chapter, verse = occurrence.split(':')
+      results.append(f"Book: {title} ({book})\nChapter: {chapter}, Verse: {verse}\nPhrase: {words}\nTranslation: {translation}\n\n")
+
+  conn.close()
+  return "\n".join(results)
+
+def run_app():
+  """Initializes and launches the Gradio app."""
+  initialize_database()
+  initialize_translator()
+
+  # Pre-populate the database
+  tanach_texts = process_json_files(1, 39)
+  populate_database(tanach_texts)
+
+  iface = gr.Interface(
     fn=gematria_search_interface,
     inputs=gr.Textbox(label="Enter phrase"),
-    outputs=[gr.Textbox(label="Results"), gr.Textbox(label="Debug Output")],
+    outputs=gr.Textbox(label="Results"),
     title="Gematria Search in Tanach",
-    description="Search for phrases in Tanach that have the same gematria value as the entered phrase.",
-    live=False,  # Disable live update
-    allow_flagging="never"  # Disable flagging 
-)
+    description="Search for phrases in the Tanach that have the same Gematria value.",
+    live=False,
+    allow_flagging="never"
+  )
+  iface.launch()
 
 if __name__ == "__main__":
-    initialize_database()
-    run_test()  # Run tests
-    iface.launch()
+  run_app()

gematria.db

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a70de31a4b4d75fe746acc35d7567274615330a7e98872ee123f9cff3a51b38c
-size 44244992
+oid sha256:6fcf5c1564217654d09ee0f57dc0025114c5df51a352f829932e86a10570ce09
+size 47960064