Add practice reports queries and libraries

dev-requirements.txt

@@ -6,12 +6,16 @@
 #
 altair==5.0.1
     # via streamlit
+annotated-types==0.5.0
+    # via pydantic
 anyio==3.7.1
     # via httpcore
 attrs==23.1.0
     # via
     #   jsonschema
     #   referencing
+beautifulsoup4==4.12.2
+    # via YFDashboard (pyproject.toml)
 black==23.7.0
     # via YFDashboard (pyproject.toml)
 blinker==1.6.2
@@ -35,6 +39,8 @@ gitpython==3.1.34
     # via streamlit
 h11==0.14.0
     # via httpcore
+html5lib==1.1
+    # via YFDashboard (pyproject.toml)
 httpcore==0.17.3
     # via httpx
 httpx==0.24.1
@@ -58,6 +64,10 @@ jsonschema==4.19.0
     # via altair
 jsonschema-specifications==2023.7.1
     # via jsonschema
+lxml==4.9.3
+    # via YFDashboard (pyproject.toml)
+lxml-stubs==0.4.0
+    # via YFDashboard (pyproject.toml)
 markdown-it-py==3.0.0
     # via rich
 markupsafe==2.1.3
@@ -103,6 +113,10 @@ protobuf==4.24.0
     # via streamlit
 pyarrow==12.0.1
     # via streamlit
+pydantic==2.3.0
+    # via YFDashboard (pyproject.toml)
+pydantic-core==2.6.3
+    # via pydantic
 pydeck==0.8.0
     # via streamlit
 pygments==2.16.1
@@ -134,7 +148,9 @@ rpds-py==0.9.2
 ruff==0.0.284
     # via YFDashboard (pyproject.toml)
 six==1.16.0
-    # via python-dateutil
+    # via
+    #   html5lib
+    #   python-dateutil
 smmap==5.0.0
     # via gitdb
 sniffio==1.3.0
@@ -142,6 +158,8 @@ sniffio==1.3.0
     #   anyio
     #   httpcore
     #   httpx
+soupsieve==2.5
+    # via beautifulsoup4
 streamlit==1.25.0
     # via
     #   YFDashboard (pyproject.toml)
@@ -158,9 +176,15 @@ tornado==6.3.3
     # via streamlit
 types-pytz==2023.3.0.1
     # via pandas-stubs
+types-requests==2.31.0.2
+    # via YFDashboard (pyproject.toml)
+types-urllib3==1.26.25.14
+    # via types-requests
 typing-extensions==4.7.1
     # via
     #   mypy
+    #   pydantic
+    #   pydantic-core
     #   streamlit
 tzdata==2023.3
     # via
@@ -174,5 +198,7 @@ validators==0.21.2
     # via streamlit
 watchdog==3.0.0
     # via streamlit
+webencodings==0.5.1
+    # via html5lib
 zipp==3.16.2
     # via importlib-metadata

pyproject.toml

@@ -7,8 +7,12 @@ version = "1"
 name = "YFDashboard"
 dependencies = [
     "streamlit",
+    "beautifulsoup4",
+    "html5lib",
+    "lxml",
     "numpy",
     "pandas",
+    "pydantic",
     "streamlit-oauth",
     ]
 
@@ -16,9 +20,11 @@ dependencies = [
 dev = [
     "black",
     "ruff",
+    "lxml-stubs",
     "mypy",
     "pytest",
     "pandas-stubs",
+    "types-requests",
     ]
 
 [tool.black]

requirements.txt

@@ -6,12 +6,16 @@
 #
 altair==5.0.1
     # via streamlit
+annotated-types==0.5.0
+    # via pydantic
 anyio==3.7.1
     # via httpcore
 attrs==23.1.0
     # via
     #   jsonschema
     #   referencing
+beautifulsoup4==4.12.2
+    # via YFDashboard (pyproject.toml)
 blinker==1.6.2
     # via streamlit
 cachetools==5.3.1
@@ -31,6 +35,8 @@ gitpython==3.1.34
     # via streamlit
 h11==0.14.0
     # via httpcore
+html5lib==1.1
+    # via YFDashboard (pyproject.toml)
 httpcore==0.17.3
     # via httpx
 httpx==0.24.1
@@ -52,6 +58,8 @@ jsonschema==4.19.0
     # via altair
 jsonschema-specifications==2023.7.1
     # via jsonschema
+lxml==4.9.3
+    # via YFDashboard (pyproject.toml)
 markdown-it-py==3.0.0
     # via rich
 markupsafe==2.1.3
@@ -79,6 +87,10 @@ protobuf==4.24.0
     # via streamlit
 pyarrow==12.0.1
     # via streamlit
+pydantic==2.3.0
+    # via YFDashboard (pyproject.toml)
+pydantic-core==2.6.3
+    # via pydantic
 pydeck==0.8.0
     # via streamlit
 pygments==2.16.1
@@ -106,7 +118,9 @@ rpds-py==0.9.2
     #   jsonschema
     #   referencing
 six==1.16.0
-    # via python-dateutil
+    # via
+    #   html5lib
+    #   python-dateutil
 smmap==5.0.0
     # via gitdb
 sniffio==1.3.0
@@ -114,6 +128,8 @@ sniffio==1.3.0
     #   anyio
     #   httpcore
     #   httpx
+soupsieve==2.5
+    # via beautifulsoup4
 streamlit==1.25.0
     # via
     #   YFDashboard (pyproject.toml)
@@ -129,7 +145,10 @@ toolz==0.12.0
 tornado==6.3.3
     # via streamlit
 typing-extensions==4.7.1
-    # via streamlit
+    # via
+    #   pydantic
+    #   pydantic-core
+    #   streamlit
 tzdata==2023.3
     # via
     #   pandas
@@ -142,5 +161,7 @@ validators==0.21.2
     # via streamlit
 watchdog==3.0.0
     # via streamlit
+webencodings==0.5.1
+    # via html5lib
 zipp==3.16.2
     # via importlib-metadata

src/domain/conferences.py

@@ -0,0 +1,17 @@
+from dataclasses import dataclass
+from typing import List
+
+
+@dataclass
+class NFLConference:
+    name: str
+    short_name: str
+
+    def __post_init__(self):
+        ALL_CONFERENCES.append(self)
+
+
+ALL_CONFERENCES: List[NFLConference] = []
+
+NFC = NFLConference(name="National Football Conference", short_name="NFC")
+AFC = NFLConference(name="American Football Conference", short_name="AFC")

src/domain/divisions.py

@@ -0,0 +1,24 @@
+from dataclasses import dataclass
+from typing import List
+from domain import conferences
+
+
+@dataclass
+class NFLDivision:
+    name: str
+    conference: conferences.NFLConference
+
+    def __post_init__(self):
+        ALL_DIVISIONS.append(self)
+
+
+ALL_DIVISIONS: List[NFLDivision] = []
+
+NFCWest = NFLDivision(name="NFC West", conference=conferences.NFC)
+NFCNorth = NFLDivision(name="NFC North", conference=conferences.NFC)
+NFCSouth = NFLDivision(name="NFC South", conference=conferences.NFC)
+NFCEast = NFLDivision(name="NFC East", conference=conferences.NFC)
+AFCWest = NFLDivision(name="AFC West", conference=conferences.AFC)
+AFCNorth = NFLDivision(name="AFC North", conference=conferences.AFC)
+AFCSouth = NFLDivision(name="AFC South", conference=conferences.AFC)
+AFCEast = NFLDivision(name="AFC East", conference=conferences.AFC)

src/domain/teams.py

@@ -0,0 +1,286 @@
+from dataclasses import dataclass
+from urllib.parse import urljoin
+from typing import List
+from domain.conferences import NFLConference
+from domain import divisions
+
+
+@dataclass
+class NFLTeam:
+    team_name: str
+    team_short_name: str
+    city: str
+    division: divisions.NFLDivision
+    footballguys_short_name: str = ""
+    url: str = ""
+    injury_report_suffix: str = "/team/injury-report/"
+
+    def __post_init__(self):
+        self.footballguys_short_name = (
+            self.team_short_name if self.footballguys_short_name == "" else self.footballguys_short_name
+        )
+        self.conference: NFLConference = self.division.conference
+        self.injury_report_url = urljoin(self.url, self.injury_report_suffix)
+        self.team_full_name = " ".join([self.city, self.team_name])
+
+        ALL_TEAMS.append(self)
+
+
+ALL_TEAMS: List[NFLTeam] = []
+
+
+arizona_cardinals = NFLTeam(
+    team_name="Cardinals",
+    team_short_name="ARI",
+    city="Arizona",
+    division=divisions.NFCWest,
+    url="https://www.azcardinals.com/",
+)
+
+atlanta_falcons = NFLTeam(
+    team_name="Falcons",
+    team_short_name="ATL",
+    city="Atlanta",
+    division=divisions.NFCSouth,
+    url="https://www.atlantafalcons.com/",
+)
+
+baltimore_ravens = NFLTeam(
+    team_name="Ravens",
+    team_short_name="BAL",
+    city="Baltimore",
+    division=divisions.AFCNorth,
+    url="https://www.baltimoreravens.com/",
+)
+
+buffalo_bills = NFLTeam(
+    team_name="Bills",
+    team_short_name="BUF",
+    city="Buffalo",
+    division=divisions.AFCEast,
+    url="https://www.buffalobills.com/",
+)
+
+carolina_panthers = NFLTeam(
+    team_name="Panthers",
+    team_short_name="CAR",
+    city="Carolina",
+    division=divisions.NFCSouth,
+    url="https://www.panthers.com/",
+)
+
+chicago_bears = NFLTeam(
+    team_name="Bears",
+    team_short_name="CHI",
+    city="Chicago",
+    division=divisions.NFCNorth,
+    url="https://www.chicagobears.com/",
+)
+
+cincinnati_bengals = NFLTeam(
+    team_name="Bengals",
+    team_short_name="CIN",
+    city="Cincinnati",
+    division=divisions.AFCNorth,
+    url="https://www.bengals.com/",
+)
+
+cleveland_browns = NFLTeam(
+    team_name="Browns",
+    team_short_name="CLE",
+    city="Cleveland",
+    division=divisions.AFCNorth,
+    url="https://www.clevelandbrowns.com/",
+)
+
+dallas_cowboys = NFLTeam(
+    team_name="Cowboys",
+    team_short_name="DAL",
+    city="Dallas",
+    division=divisions.NFCEast,
+    url="https://www.dallascowboys.com/",
+)
+
+denver_broncos = NFLTeam(
+    team_name="Broncos",
+    team_short_name="DEN",
+    city="Denver",
+    division=divisions.AFCWest,
+    url="https://www.denverbroncos.com/",
+)
+
+detroit_lions = NFLTeam(
+    team_name="Lions",
+    team_short_name="DET",
+    city="Detroit",
+    division=divisions.NFCNorth,
+    url="https://www.detroitlions.com/",
+)
+
+green_bay_packers = NFLTeam(
+    team_name="Packers",
+    team_short_name="GB",
+    city="Green Bay",
+    division=divisions.NFCNorth,
+    url="https://www.packers.com/",
+)
+
+houston_texans = NFLTeam(
+    team_name="Texans",
+    team_short_name="HOU",
+    city="Houston",
+    division=divisions.AFCSouth,
+    url="https://www.houstontexans.com/",
+)
+
+indianapolis_colts = NFLTeam(
+    city="Indianapolis",
+    team_name="Colts",
+    team_short_name="IND",
+    division=divisions.AFCSouth,
+    url="https://www.colts.com/",
+)
+
+jacksonville_jaguars = NFLTeam(
+    city="Jacksonville",
+    team_name="Jaguars",
+    team_short_name="JAX",
+    division=divisions.AFCSouth,
+    url="https://www.jaguars.com/",
+)
+
+kansas_city_chiefs = NFLTeam(
+    city="Kansas City",
+    team_name="Chiefs",
+    team_short_name="KC",
+    division=divisions.AFCWest,
+    url="https://www.chiefs.com/",
+)
+
+las_vegas_raiders = NFLTeam(
+    city="Las Vegas",
+    team_name="Raiders",
+    team_short_name="LV",
+    division=divisions.AFCWest,
+    url="https://www.raiders.com/",
+)
+
+los_angeles_chargers = NFLTeam(
+    city="Los Angeles",
+    team_name="Chargers",
+    team_short_name="LAC",
+    division=divisions.AFCWest,
+    url="https://www.chargers.com/",
+)
+
+los_angeles_rams = NFLTeam(
+    city="Los Angeles",
+    team_name="Rams",
+    team_short_name="LAR",
+    division=divisions.NFCWest,
+    url="https://www.therams.com/",
+)
+
+miami_dolphins = NFLTeam(
+    city="Miami",
+    team_name="Dolphins",
+    team_short_name="MIA",
+    division=divisions.AFCEast,
+    url="https://www.miamidolphins.com/",
+)
+
+minnesota_vikings = NFLTeam(
+    city="Minnesota",
+    team_name="Vikings",
+    team_short_name="MIN",
+    division=divisions.NFCNorth,
+    url="https://www.vikings.com/",
+)
+
+new_england_patriots = NFLTeam(
+    city="New England",
+    team_name="Patriots",
+    team_short_name="NE",
+    division=divisions.AFCEast,
+    url="https://www.patriots.com/",
+)
+
+new_orleans_saints = NFLTeam(
+    city="New Orleans",
+    team_name="Saints",
+    team_short_name="NO",
+    division=divisions.NFCSouth,
+    url="https://www.neworleanssaints.com/",
+)
+
+new_york_giants = NFLTeam(
+    city="New York",
+    team_name="Giants",
+    team_short_name="NYG",
+    division=divisions.NFCEast,
+    url="https://www.giants.com/",
+)
+
+new_york_jets = NFLTeam(
+    city="New York",
+    team_name="Jets",
+    team_short_name="NYJ",
+    division=divisions.AFCEast,
+    url="https://www.newyorkjets.com/",
+)
+
+philadelphia_eagles = NFLTeam(
+    city="Philadelphia",
+    team_name="Eagles",
+    team_short_name="PHI",
+    division=divisions.NFCEast,
+    url="https://www.philadelphiaeagles.com/",
+)
+
+pittsburgh_steelers = NFLTeam(
+    city="Pittsburgh",
+    team_name="Steelers",
+    team_short_name="PIT",
+    division=divisions.AFCNorth,
+    url="https://www.steelers.com/",
+)
+
+san_francisco_49ers = NFLTeam(
+    city="San Francisco",
+    team_name="49ers",
+    team_short_name="SF",
+    division=divisions.NFCWest,
+    url="https://www.49ers.com/",
+)
+
+seattle_seahawks = NFLTeam(
+    city="Seattle",
+    team_name="Seahawks",
+    team_short_name="SEA",
+    division=divisions.NFCWest,
+    url="https://www.seahawks.com/",
+)
+
+tampa_bay_buccaneers = NFLTeam(
+    city="Tampa Bay",
+    team_name="Buccaneers",
+    team_short_name="TB",
+    division=divisions.NFCSouth,
+    url="https://www.buccaneers.com/",
+)
+
+tennessee_titans = NFLTeam(
+    city="Tennessee",
+    team_name="Titans",
+    team_short_name="TEN",
+    division=divisions.AFCSouth,
+    url="https://www.tennesseetitans.com/",
+)
+
+washington_football_team = NFLTeam(
+    city="Washington",
+    team_name="Commanders",
+    team_short_name="WAS",
+    division=divisions.NFCEast,
+    url="https://www.commanders.com/",
+)

src/queries/nfl_teams/practice_reports.py

@@ -0,0 +1,109 @@
+import datetime
+from multiprocessing import Pool
+import numpy as np
+import pandas as pd
+from pydantic import BaseModel, Field
+from typing import Optional
+from urllib.parse import urljoin
+
+from domain.teams import ALL_TEAMS, NFLTeam
+
+
+MULTIPROCESSING_ENABLED = False
+
+PRACTICE_WEEK = {
+    "Mon": 0,
+    "Tue": 1,
+    "Wed": 2,
+    "Thu": 3,
+    "Fri": 4,
+    "Sat": 5,
+    "Sun": 6,
+    "Monday": 0,
+    "Tuesday": 1,
+    "Wednesday": 2,
+    "Thursday": 3,
+    "Friday": 4,
+    "Saturday": 5,
+    "Sunday": 6,
+}
+
+
+DAY_OF_WEEK_STRING_MAPPING = {
+    "Monday": "Mon",
+    "Tuesday": "Tue",
+    "Wednesday": "Wed",
+    "Thursday": "Thu",
+    "Friday": "Fri",
+    "Saturday": "Sat",
+    "Sunday": "Sun",
+}
+
+
+WEEK_1_BEGIN_DATE = datetime.datetime(2023, 9, 4)
+CURRENT_DATE = datetime.datetime.now()
+CURRENT_WEEK = max(1, int(1 + (CURRENT_DATE - WEEK_1_BEGIN_DATE).days / 7))
+
+
+class PracticeReportRawRow(BaseModel):
+    Team: str
+    Player: str
+    Position: str
+    Injury: str
+    Sun: Optional[str] = None
+    Mon: Optional[str] = None
+    Tue: Optional[str] = None
+    Wed: Optional[str] = None
+    Thu: Optional[str] = None
+    Fri: Optional[str] = None
+    Sat: Optional[str] = None
+    game_status: str = Field(alias="Game Status")
+
+    @classmethod
+    def replace_nan(self, value) -> str:
+        if isinstance(value, float):
+            if np.isnan(value):
+                return ""
+        return value
+
+    @classmethod
+    def from_raw(cls, input_dict) -> "PracticeReportRawRow":
+        return cls(**{DAY_OF_WEEK_STRING_MAPPING.get(k, k): cls.replace_nan(v) for k, v in input_dict.items()})
+
+
+def scrape_team_injury_report(team: NFLTeam) -> pd.DataFrame:
+    print(f"Scraping Injury Report for: {team.team_full_name}")
+    injury_report_url = urljoin(team.injury_report_url, f"week/REG-{CURRENT_WEEK}")
+    team_report = pd.read_html(injury_report_url)[0]
+    try:
+        team_report = pd.read_html(injury_report_url)[0]
+    except Exception:
+        print(f"Failed to scrape practice report for: {team.team_full_name}")
+        return pd.DataFrame()
+    validated_row_list = []
+    for df_row_dict in team_report.to_dict("records"):
+        row_to_add = df_row_dict
+        row_to_add["Team"] = team.team_full_name
+        validated_row_list.append(PracticeReportRawRow.from_raw(row_to_add))
+    validated_df = pd.DataFrame([x.dict() for x in validated_row_list])
+    # drop all na columns
+    validated_df.dropna(axis=1, how="all", inplace=True)
+    # replace day of week with practice day from 1-3
+    day_idx = 1
+    last_practice_day = None
+    for col in validated_df.columns:
+        if col in PRACTICE_WEEK:
+            validated_df.rename(columns={col: str(day_idx)}, inplace=True)
+            day_idx += 1
+            last_practice_day = col
+    validated_df["Last Practice Day"] = last_practice_day
+    return validated_df
+
+
+def scrape_all_team_injury_report() -> pd.DataFrame:
+    if MULTIPROCESSING_ENABLED:
+        with Pool() as pool:
+            team_df_list = pool.map(scrape_team_injury_report, ALL_TEAMS)
+    else:
+        team_df_list = [scrape_team_injury_report(team) for team in ALL_TEAMS]
+    return pd.concat(team_df_list)