Add copy loader page that loads dbt

src/dbt_data_client.py

@@ -1,8 +1,23 @@
 import duckdb
 import os
+import subprocess
+import streamlit as st
 
-DUCKDB_PATH = os.path.join(os.path.dirname(__file__), "dbt_yfdash", "dev.duckdb")
+
+DBT_DIR = os.path.join(os.path.dirname(__file__), "dbt_yfdash")
+DUCKDB_PATH = os.path.join(DBT_DIR, "dev.duckdb")
 
 
 def get_db_conn():
     return duckdb.connect(DUCKDB_PATH)
+
+
+def run_dbt_build():
+    process = subprocess.Popen(
+        args=["dbt", "build"],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        cwd=DBT_DIR,
+    )
+    stdout, _ = process.communicate()
+    st.text("\n".join(stdout.decode().split("\n")[1:][:-1]))

src/pages/98_Load_Data copy.py

@@ -0,0 +1,38 @@
+import streamlit as st
+
+from config import DEFAULT_ICON
+from shared_page import common_page_config
+
+from queries.nflverse.github_data import get_current_tables
+
+from dbt_data_client import get_db_conn, run_dbt_build
+
+
+def get_page():
+    page_title = "Data Loader - dbt"
+    st.set_page_config(page_title=page_title, page_icon=DEFAULT_ICON, layout="wide")
+    common_page_config()
+    st.title(page_title)
+
+    with get_db_conn() as duckdb_conn:
+        current_tables_list = get_current_tables(duckdb_conn)
+
+    if st.button("Refresh Data"):
+        run_dbt_build()
+
+    if selected_table := st.selectbox("Describe a table:", current_tables_list, index=0):
+        with get_db_conn() as duckdb_conn:
+            describe_df = duckdb_conn.sql(f"DESCRIBE {selected_table}").df()
+        st.dataframe(
+            describe_df,
+            hide_index=True,
+            use_container_width=True,
+        )
+
+        if st.checkbox("Explore data"):
+            with get_db_conn() as duckdb_conn:
+                st.dataframe(duckdb_conn.sql(f"SELECT * FROM {selected_table} LIMIT 50").df())
+
+
+if __name__ == "__main__":
+    get_page()

src/queries/nflverse/github_data.py

@@ -140,6 +140,6 @@ def load_assets():
             st.write(f"{tag} failed to load")
 
 
-def get_current_tables() -> list[str]:
-    current_tables_df = duckdb.sql("SHOW TABLES").df()
+def get_current_tables(duckdb_conn=duckdb) -> list[str]:
+    current_tables_df = duckdb_conn.sql("SHOW TABLES").df()
     return current_tables_df["name"].tolist()