Add load data page using duckdb

src/Home.py

@@ -4,6 +4,8 @@ from config import DEFAULT_ICON, LEAGUE_NAME
 from login_component import get_authorization_button
 from page_selector import remove_seasonal_pages
 
+from queries.nflverse.github_data import load_assets
+
 
 def get_app():
     keeper_title = f"{LEAGUE_NAME}"
@@ -24,4 +26,5 @@ def get_app():
 
 
 if __name__ == "__main__":
+    load_assets()
     get_app()

src/pages/98_Load_Data.py

@@ -0,0 +1,32 @@
+import duckdb
+import streamlit as st
+
+from config import DEFAULT_ICON
+from shared_page import common_page_config
+
+from queries.nflverse.github_data import load_assets
+
+
+def get_page():
+    page_title = "Data Loader"
+    st.set_page_config(page_title=page_title, page_icon=DEFAULT_ICON, layout="wide")
+    common_page_config()
+    st.title(page_title)
+
+    current_tables_df = duckdb.sql("SHOW TABLES").df()
+    current_tables_list = current_tables_df["name"].tolist()
+
+    if st.button("Refresh Data"):
+        load_assets()
+
+    if selected_table := st.selectbox("Describe a table:", current_tables_list, index=0):
+        describe_df = duckdb.sql(f"DESCRIBE {selected_table}").df()
+        st.dataframe(
+            describe_df,
+            hide_index=True,
+            use_container_width=True,
+        )
+
+
+if __name__ == "__main__":
+    get_page()

src/queries/nflverse/github_data.py

@@ -1,4 +1,10 @@
+import duckdb
 import pandas as pd
+import os
+from typing import Callable
+
+
+BASE_URL = "https://github.com/nflverse/nflverse-data/releases/download/"
 
 
 def get_parquet_github(season_int: int, parquet_prefix: str):
@@ -18,7 +24,7 @@ FANTASY_POSITIONS = [
 
 
 def get_snap_counts(season_int: int) -> pd.DataFrame:
-    df = get_parquet_github(season_int, "snap_counts/snap_counts")
+    df = duckdb.sql(f"SELECT * from snap_counts_snap_counts_{season_int}").df()
     df["fantasy_position"] = df["position"].isin(FANTASY_POSITIONS)
     return df
 
@@ -37,3 +43,70 @@ def get_ftn_charting(season_int: int) -> pd.DataFrame:
 
 def get_pbp_participation(season_int: int) -> pd.DataFrame:
     return get_parquet_github(season_int, "pbp_participation/pbp_participation")
+
+
+SEASON = "2023"
+
+NFLVERSE_ASSETS = [
+    ("ftn_charting", f"ftn_charting_{SEASON}.parquet"),
+    ("espn_data", "qbr_season_level.parquet"),
+    ("espn_data", "qbr_week_level.parquet"),
+    ("players", "players.parquet"),
+    ("pbp_participation", f"pbp_participation_{SEASON}.parquet"),
+    ("snap_counts", f"snap_counts_{SEASON}.parquet"),
+    ("player_stats", f"player_stats_{SEASON}.parquet"),
+    ("player_stats", f"player_stats_def_{SEASON}.parquet"),
+    ("player_stats", f"player_stats_kicking_{SEASON}.parquet"),
+    ("pfr_advstats", "advstats_season_def.parquet"),
+    ("pfr_advstats", "advstats_season_pass.parquet"),
+    ("pfr_advstats", "advstats_season_rec.parquet"),
+    ("pfr_advstats", "advstats_season_rush.parquet"),
+    ("pfr_advstats", f"advstats_week_def_{SEASON}.parquet"),
+    ("pfr_advstats", f"advstats_week_pass_{SEASON}.parquet"),
+    ("pfr_advstats", f"advstats_week_rec_{SEASON}.parquet"),
+    ("pfr_advstats", f"advstats_week_rush_{SEASON}.parquet"),
+    ("pbp", f"play_by_play_{SEASON}.parquet"),
+    ("nextgen_stats", "ngs_passing.parquet"),
+    ("nextgen_stats", "ngs_receiving.parquet"),
+    ("nextgen_stats", "ngs_rushing.parquet"),
+]
+
+
+class NflVerseDataAsset:
+    def __init__(
+        self,
+        release_tag: str,
+        asset_name: str,
+        dataframe_mutation_fxn: Callable[[pd.DataFrame], pd.DataFrame] = lambda x: x,
+    ):
+        self.release_tag = release_tag
+        self.asset_name = asset_name
+        self.dataframe_mutation_fxn = dataframe_mutation_fxn
+        self.table_name = f"{release_tag}_{asset_name.rsplit('.', 1)[0]}"
+
+    def load_parquet_asset_to_df(self) -> pd.DataFrame:
+        location = os.path.join(BASE_URL, self.release_tag, self.asset_name)
+        df = pd.read_parquet(location)
+        return df
+
+    def register_asset_to_duckdb(self) -> None:
+        df = self.load_parquet_asset_to_df()
+        df = self.dataframe_mutation_fxn(df)
+        duckdb.register(self.table_name, df)
+
+
+def load_assets():
+    for tag, asset in NFLVERSE_ASSETS:
+        asset = NflVerseDataAsset(tag, asset)
+        asset.register_asset_to_duckdb()
+
+
+def main():
+    load_assets()
+    import pdb
+
+    pdb.set_trace()
+
+
+if __name__ == "__main__":
+    main()