Update player name translation, add daily and weekly leaderboards

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,13 +1,17 @@
 import gradio as gr
+import matplotlib as mpl
 
 from data import data_df
 from pitcher_overview import create_pitcher_overview
 from pitch_leaderboard import create_pitch_leaderboard
+from daily_weekly_leaderboard import create_daily_weekly_leaderboard_app
 from css import css
 
+mpl.use('Agg')
+
 updated = '2025-07-21'
 limitations = '''**General Limitations**
-- Foreign players names are in Hebpurn romanization. Contact me if you need a card for a foreign player.
+- As new players make their debut, some names may not be translated/transliterated correctly.
 '''
 
 if __name__ == '__main__':
@@ -16,6 +20,8 @@ if __name__ == '__main__':
       create_pitcher_overview(data_df)
     with gr.Tab('Pitch Leaderboard'):
       create_pitch_leaderboard()
+    with gr.Tab('Daily/Weekly Leaderboard'):
+      create_daily_weekly_leaderboard_app(data_df)
 
     gr.Markdown(f'Last updated: {updated}')
     gr.Markdown(limitations)

daily_weekly_leaderboard.py

@@ -0,0 +1,83 @@
+import gradio as gr
+import matplotlib.pyplot as plt
+
+import datetime
+
+from plotting import create_daily_weekly_leaderboard
+from data import data_df
+
+
+def gr_create_daily_weekly_leaderboards(leaderboard_date, whiff_leaders, velo_leaders, data_df):
+
+  filenames = []
+  for time_type in ('daily', 'weekly'):
+    for stat, leaders in zip(('whiff', 'velo'), (whiff_leaders, velo_leaders)):
+      create_daily_weekly_leaderboard(stat, leaderboard_date, time_type, leaders, data_df)
+      filename = f'{stat}_{time_type}.png'
+      plt.savefig(filename, bbox_inches='tight')
+      filenames.append(filename)
+
+  return filenames
+
+
+def go_back_day(date):
+  return date - datetime.timedelta(days=1)
+
+
+def go_forward_day(date):
+  return date + datetime.timedelta(days=1)
+
+
+def go_back_week(date):
+  return date - datetime.timedelta(days=7)
+
+
+def go_forward_week(date):
+  return date + datetime.timedelta(days=7)
+
+
+def create_daily_weekly_leaderboard_app(data_df):
+  with gr.Blocks() as app:
+    gr.Markdown('# Daily/Weekly Leaderboards')
+    
+    _data_df = gr.State(data_df)
+
+    with gr.Row():
+      date_init = data_df['date'].max()
+      date_init = datetime.datetime(date_init.year, date_init.month, date_init.day)
+      leaderboard_date =  gr.DateTime(date_init, include_time=False, type='datetime', label='Date')
+
+      whiff_leaders = gr.Number(10, precision=0, minimum=0, label='Whiff Leaders')
+      velo_leaders = gr.Number(10, precision=0, minimum=0, label='Velo Leaders')
+      search = gr.Button('Search')
+
+    with gr.Row():
+      prev_week = gr.Button('Previous Week')
+      prev_day = gr.Button('Previous Day')
+      next_day = gr.Button('Next Day')
+      next_week = gr.Button('Next Week')
+      
+    leaderboards = []
+    for time_type in ('Daily', 'Weekly'):
+      with gr.Row():
+        gr.Markdown(f'## {time_type}')
+      with gr.Row():
+        for stat in ('Whiff', 'Velo'):
+          leaderboards.append(gr.Image(label=f'{time_type} {stat} Leaderboard', height=512))
+
+    
+    search.click(gr_create_daily_weekly_leaderboards, inputs=[leaderboard_date, whiff_leaders, velo_leaders,  _data_df], outputs=leaderboards)
+
+    for btn, fn in (
+      (prev_day, go_back_day),
+      (next_day, go_forward_day),
+      (prev_week, go_back_week),
+      (next_week, go_forward_week)
+    ):
+      btn.click(fn, inputs=leaderboard_date, outputs=leaderboard_date)
+    
+  return app
+
+if __name__ == '__main__':
+  app = create_daily_weekly_leaderboard_app(data_df)
+  app.launch()

data.py

@@ -3,6 +3,9 @@ import os
 from tqdm.auto import tqdm
 import pykakasi
 from huggingface_hub import snapshot_download
+import numpy as np
+
+from string import ascii_letters
 
 from convert import (
   aux_global_id_to_code, presult, 
@@ -44,44 +47,70 @@ for season in tqdm(SEASONS):
   _aux_sched_df = pl.read_parquet(os.path.join(DATA_PATH, str(season), 'aux_schedule.parquet'))
   aux_sched_df = pl.concat((aux_sched_df, _aux_sched_df))
 
-players_df = pl.read_parquet(os.path.join(DATA_PATH, 'players.parquet'))
-kana_df = pl.read_parquet(os.path.join(DATA_PATH, 'players_kana.parquet'))
 
+def select_name(names):
+  '''
+  When given mutiple names, 
+  prioritizes the name with ASCII characters (ex. R. マルティネス > マルティネス), 
+  followed by the shorter name (ex. 大勢 > 翁田 大勢)
+  Names with ASCII characters help differentiate between foreign players,
+  whlie shorter names are more accurate for players going by shorter names
+  '''
+  lens = []
+  for name in names:
+    if any([char in ascii_letters for char in name]):
+      return name
+    else:
+      lens.append(len(name))
+  return names[np.argmin(lens).item()]
+
+# load player dfs
+players_df = (
+    pl.read_parquet('files/players.parquet')
+    .with_columns(pl.col('playerName').str.normalize('NFKC').str.replace_all('・', ' '))
+    .group_by('playerId').agg(pl.col('playerName').map_elements(select_name, return_dtype=pl.String))
+)
+translated_df = (
+    pl.read_parquet('files/players_translated.parquet')
+    .with_columns(pl.col('name_jp').str.normalize('NFKC').str.replace_all('・', ' '))
+)
+manual_translated_df = pl.read_parquet('files/players_translated_manual.parquet')
+
+# names with no romanization are approximated with kana translation
 kks = pykakasi.kakasi()
-kana_df = (
-    kana_df
+
+# take names in parenthesis when they contain an ascii character
+translated_df = (
+    translated_df
     .with_columns(
-        pl.col('name').str.normalize('NFKC'),
-        (
-            pl.col('name_kana')
-            .map_elements(
-                lambda name: ''.join([word['hepburn'].capitalize() for word in kks.convert(name)]),
-                return_dtype=pl.String
+        pl.when(pl.col('name_jp').str.contains(r'\('))
+        .then(pl.col('name_jp').str.extract(r'.*\(', 0).str.strip_chars_end(' ('))
+        .otherwise(pl.col('name_jp'))
+        .str.replace_all('・', ' ')
+        .alias('name_jp')
+    )
+    .with_columns(pl.col('name_kana').str.normalize('NFKC').str.replace_all('・', ' '))
+    .with_columns(pl.col('name_kana').str.extract(r'\(.*\)', 0).str.strip_chars('()').alias('in_parentheses'))
+    .with_columns(pl.col('name_kana').str.extract(r'.*\(', 0).str.strip_chars_end('(').alias('before_parentheses'))
+    .with_columns(
+        pl.when(pl.col('name_en').is_null())
+        .then
+         (
+            pl.when(pl.col('in_parentheses').is_not_null() | pl.col('before_parentheses').is_not_null())
+            .then(
+                pl.when(pl.col('in_parentheses').map_elements(lambda name: any([char in ascii_letters for char in name]), pl.Boolean))
+                .then(pl.col('in_parentheses'))
+                .otherwise(pl.col('before_parentheses'))
             )
-            .alias('name_en')
+            .otherwise(pl.col('name_kana').map_elements(lambda name: ''.join([word['hepburn'].capitalize() for word in kks.convert(name)]), return_dtype=pl.String))
         )
+        .otherwise(pl.col('name_en'))
+        .alias('name_en')
     )
-    .with_columns(pl.col('name_en').str.to_lowercase())
+    .with_columns(pl.col('name_en').str.replace_all(',', '').str.to_titlecase())
 )
 
-for old_part, new_part in [
-  ('you', 'yo'),
-  ('kou', 'ko'),
-  ('gou', 'go'),
-  ('shou', 'sho'),
-  ('jou', 'jo'),
-  ('rou', 'ro'),
-  ('ou', 'oh'),
-  ('shuu', 'shu'),
-  ('ryuu', 'ryu'),
-  ('yuu', 'yu'),
-  ('oo', 'o') # messes with someone whose name ends in koo
-]:
-  kana_df = kana_df.with_columns(pl.col('name_en').str.replace(old_part, new_part))
-
-kana_df = kana_df.with_columns(pl.col('name_en').str.to_titlecase())
-
-players_df = players_df.with_columns(pl.col('playerName').str.normalize('NFKC'))
+# handle inconsistent kanji between sources
 for old_char, new_char in [
     ('崎', '﨑'),
     ('高', '髙'),
@@ -91,13 +120,33 @@ for old_char, new_char in [
 ]:
   players_df = (
       players_df.with_columns(
-          pl.when(~pl.col('playerName').is_in(kana_df['name']))
+          pl.when(~pl.col('playerName').is_in(translated_df['name_jp']))
           .then(pl.col('playerName').str.replace(old_char, new_char))
           .otherwise('playerName')
       )
   )
 
-players_df = players_df.join(kana_df, left_on='playerName', right_on='name', how='left')
+# merge player dfs
+players_df = (
+    players_df
+    .join(manual_translated_df.rename({'name_en': 'name_en_manual'}), on='playerId', how='left')
+    .join(
+        (
+            translated_df
+            .with_columns(
+                pl.when(pl.col('name_jp').str.contains(r'\.') & ~pl.col('name_jp').is_in(players_df.filter(pl.len().over('playerName') == 1)['playerName']))
+                .then(pl.col('name_jp').str.strip_chars(ascii_letters+'.'))
+                .otherwise('name_jp')
+            )
+            [['name_jp', 'name_en']]
+        ),
+        left_on='playerName', right_on='name_jp', how='left'
+    )
+    .with_columns(pl.coalesce('name_en_manual', 'name_en').alias('name_en'))
+    .unique() # remove duplicates from names with multiple matches in other dataframes
+    .drop('name_en_manual', 'name_jp')
+    # .filter(pl.col('name_en').is_null())
+)
 
 aux_df = (
     aux_df

pitch_leaderboard.py

@@ -139,7 +139,7 @@ def create_pitch_leaderboard():
       all_teams = gr.Button('Select/Deselect all teams')
         
     search = gr.Button('Search')
-    # pin_columns = gr.Checkbox(True, 'Pin columns')
+    pin_columns = gr.Button('Pin columns')
     leaderboard = gr.DataFrame(
       pl.DataFrame({'Pitcher': [], 'Pitch': []}),
       column_widths=[125, 75, 125, 125] + [max(50, 10*len(stat)) for stat in STATS],
@@ -159,6 +159,11 @@ def create_pitch_leaderboard():
       # inputs=pin_columns,
       # outputs=leaderboard
     # )
+    pin_columns.click(
+      lambda : gr.update(pinned_columns=None),
+      # inputs=pin_columns,
+      outputs=leaderboard
+    )
     
   return app
 

pitcher_overview.py

@@ -10,7 +10,6 @@ notes = '''**Limitations**
 - Only supports regular season data
 
 **To-do**
-- Fix names of foreign players
 - Add teams insignias
 - Measure percentiles per pitcher handedness
 - Allow for arbitrary date ranges

plotting.py

@@ -6,15 +6,15 @@ import polars as pl
 from pyfonts import load_google_font
 from scipy.stats import gaussian_kde
 import numpy as np
+from PIL import Image
 
 from types import SimpleNamespace
+import datetime
 from datetime import date
+import os
 
-from convert import ball_kind_code_to_color, get_text_color_from_color
-from stats import get_pitcher_stats
-
-
-mpl.use('Agg')
+from convert import ball_kind_code_to_color, get_text_color_from_color, team_names_short_to_color, get_text_color_from_team
+from stats import get_pitcher_stats, filter_data_by_date_and_game_kind
 
 
 def get_card_data(id, **kwargs):
@@ -243,3 +243,165 @@ def create_pitcher_overview_card(id, season, dpi=300):
   return fig
 # fig = create_card('1600153', season=2023, dpi=300)
 # plt.show()
+
+# DAILY/WEEKLY LEADERBOARDS
+
+def get_whiff_leaderboard_data(data, leaders, include_date):
+  data = (
+      data
+      .group_by('pitId', 'pitcher_team_name_short', 'date')
+      .agg(
+          pl.col('pitcher_name').first(),
+          pl.col('whiff').sum().alias('Whiffs')
+      )
+      .sort(['Whiffs', 'pitcher_name'], descending=[True, False])
+  )
+  # if len(data) > 0:
+    # data = data.filter(pl.col('Whiffs') >= data['Whiffs'][min(leaders, len(data)-1)])
+  data = (
+      data  
+      .rename({'pitcher_name': 'Player', 'pitcher_team_name_short': 'Team'})
+      .with_columns(
+          pl.col('date').dt.to_string('%m.%d').alias('Date'),
+          pl.col('Whiffs').rank(descending=True, method='min').alias('Rank')
+      )
+      [['Rank', 'Team', 'Player'] + (['Date'] if include_date else []) + ['Whiffs']]
+      # .with_row_index('Rank', 1)
+  )
+  # data = data.filter(pl.col('Rank') <= leaders)
+  data = data.filter(pl.col('Rank') <= data.group_by('Rank').agg(pl.len()).sort('Rank').filter(pl.col('len').cum_sum()>=leaders)['Rank'].min())
+  return data
+
+  
+def get_velo_leaderboard_data(data, leaders):
+  data = data.sort(['ballSpeed', 'pitcher_name'], descending=[True, False])
+  # if len(data) > 0:
+    # data = data.filter(pl.col('ballSpeed') >= data['ballSpeed'][min(leaders, len(data)-1)])
+  data = (
+      data
+      .rename({'ballSpeed': 'KPH', 'pitcher_name': 'Player', 'pitcher_team_name_short': 'Team'})
+      # .with_row_index('Rank', 1)
+      .with_columns(
+          (pl.col('KPH') / 1.609).round(1).alias('MPH'),
+          pl.col('KPH').rank(descending=True, method='min').alias('Rank')
+      )
+      [['Rank', 'Team', 'Player', 'KPH', 'MPH']]
+      
+  )
+  # data = data.filter(pl.col('Rank') <= leaders)
+  data = data.filter(pl.col('Rank') <= data.group_by('Rank').agg(pl.len()).sort('Rank').filter(pl.col('len').cum_sum()>=leaders)['Rank'].min())
+  return data
+
+
+def create_daily_weekly_leaderboard(stat, leaderboard_date, time_type, leaders, data):
+
+  font = load_google_font('Saira Extra Condensed', weight='medium')
+  bold_font = load_google_font('Saira Extra Condensed', weight='bold')
+  date_font = load_google_font('Lekton', weight='bold')
+
+  assert stat in ('velo', 'whiff')
+  assert time_type in ('daily', 'weekly')
+
+  if time_type == 'daily':
+    data = filter_data_by_date_and_game_kind(data, start_date=leaderboard_date, end_date=leaderboard_date)
+  else:
+    monday = leaderboard_date - datetime.timedelta(days=leaderboard_date.weekday())
+    sunday = leaderboard_date + datetime.timedelta(days=6-leaderboard_date.weekday())
+    data = filter_data_by_date_and_game_kind(data, start_date=monday, end_date=sunday)
+  
+  leaderboard = get_velo_leaderboard_data(data, leaders) if stat == 'velo' else get_whiff_leaderboard_data(data, leaders, include_date=time_type=='weekly')
+  stats = [col for col in leaderboard.columns if col not in ['Rank', 'Team', 'Player']]
+  stat_col_lens = [1 if max(leaderboard[stat].cast(pl.String).str.len_chars().max() or 0, len(stat)) < 5 else 1.5 for stat in stats]
+
+  dpi = 300
+
+  fig = plt.figure(figsize=(1080/300, 1350/300), dpi=dpi)
+  gs = fig.add_gridspec(
+    max(len(leaderboard), 1)+2, 
+    3+len(stats), 
+    height_ratios=[1] + ([9/(len(leaderboard)+1)] * (len(leaderboard)+1) if len(leaderboard) else [1, 8]),
+    width_ratios=[1, 1, 8-sum(stat_col_lens)] + stat_col_lens
+  )
+
+  data_offset = 2
+
+  axs = []
+  def create_and_add_subplot(indexed_gs):
+    ax = fig.add_subplot(indexed_gs)
+    axs.append(ax)
+    return ax
+
+  title_ax = create_and_add_subplot(gs[0, :])
+  title_ax.text(0, 0.1, f'{time_type.upper()} {stat.upper()} LEADERBOARD', verticalalignment='baseline', font=bold_font, size=15)
+  
+  if time_type == 'daily':
+    title_ax.text(1, 0.1, leaderboard_date.strftime('%Y.%m.%d (%a)'), verticalalignment='baseline', horizontalalignment='right', font=date_font, size=7)
+  else:
+
+    title_ax.text(1, 0.1, monday.strftime('%Y.%m.%d (%a)')+'\n-'+sunday.strftime('%Y.%m.%d (%a)'), verticalalignment='baseline', horizontalalignment='right', font=date_font, size=7)
+
+  rank_ax = create_and_add_subplot(gs[data_offset-1, 0])
+  rank_ax.text(0.5, 0, 'RANK', verticalalignment='bottom', horizontalalignment='center', font=bold_font)
+
+  team_ax = create_and_add_subplot(gs[data_offset-1, 1])
+  team_ax.text(0.5, 0, 'TEAM', verticalalignment='bottom', horizontalalignment='center', font=bold_font)
+
+  player_ax = create_and_add_subplot(gs[data_offset-1, 2])
+  player_ax.text(0, 0, 'PLAYER', verticalalignment='bottom', font=bold_font)
+
+  for col, stat in enumerate(stats, start=3):
+    stat_ax = create_and_add_subplot(gs[data_offset-1, col])
+    stat_ax.text(0.5, 0, stat.upper(), verticalalignment='bottom', horizontalalignment='center', font=bold_font)
+
+  midline_ax = create_and_add_subplot(gs[data_offset-1, :])
+  midline_ax.add_patch(plt.Rectangle((0, 0), 1, 0.01, color='black'))
+
+  if len(leaderboard) == 0:
+    blank_ax = create_and_add_subplot(gs[data_offset:])
+
+  for i, row in enumerate(leaderboard.iter_rows()):
+    rank, team, player, *stats = row
+    rank_ax = create_and_add_subplot(gs[i+data_offset, 0])
+    rank_ax.text(0.5, 0.5, rank, verticalalignment='center_baseline', horizontalalignment='center', font=font)
+
+    team_ax = create_and_add_subplot(gs[i+data_offset, 1])
+    image = Image.open(os.path.join('assets', 'white_insignias', f'{team.lower()}.png'))
+
+    w, h = image.size
+    new_longer_side = 512
+    if w > h:
+      w, h = (new_longer_side, round(h*new_longer_side/w))
+    else:
+      w, h = (round(w*new_longer_side/h), new_longer_side)
+    image = image.resize((w, h))
+    ax_s = 512*1.5
+    team_ax.set_xlim(0, ax_s)
+    team_ax.set_ylim(0, ax_s)
+    image = np.array(image)
+    circle = plt.Circle((ax_s/2, ax_s/2), radius=ax_s/2, color=team_names_short_to_color[team], clip_on=False, zorder=1)
+    team_ax.add_patch(circle)
+    team_ax.imshow(
+        image[..., -1],
+        cmap=LinearSegmentedColormap.from_list('tmp', [team_names_short_to_color[team], 'black' if team in ('Lotte', 'Hanshin') else 'white']),
+        extent=((ax_s-w)/2, ax_s-(ax_s-w)/2, (ax_s-h)/2, ax_s-(ax_s-h)/2),
+        zorder=2
+    )
+
+    player_ax = create_and_add_subplot(gs[i+data_offset, 2])
+    player_ax.text(0.02, 0.5, player.upper(), verticalalignment='center_baseline', font=font, color=get_text_color_from_team(team))
+    player_ax.add_patch(plt.Polygon([(0, 0), (0.98, 0), (1, 0.5), (1, 1), (0, 1)], color=team_names_short_to_color[team], clip_on=False))
+
+    for col, stat in enumerate(stats, start=3):
+      stat_ax = create_and_add_subplot(gs[i+data_offset, col])
+      stat_ax.text(0.5, 0.5, stat, verticalalignment='center_baseline', horizontalalignment='center', font=font)
+    
+  for ax in axs:
+    ax.axis('off')
+    ax.tick_params(
+        axis='both',
+        which='both',
+        length=0,
+        labelbottom=False,
+        labelleft=False
+    )
+  return fig

stats.py

@@ -35,10 +35,18 @@ def compute_team_games(data):
               .rename({'VisitorTeamNameES': 'team'})
           ),
           on='team',
+          how='full'
+      )
+      .fill_null(0)
+      .with_columns(
+        (pl.col('home_games')+pl.col('visitor_games')).alias('games'),
+        pl.when(pl.col('team').is_null())
+        .then(pl.col('team_right'))
+        .otherwise(pl.col('team')).alias('team')
       )
-      .with_columns((pl.col('home_games')+pl.col('visitor_games')).alias('games'))
   )
 
+
   return (
       data
       .drop('home_games', 'visitor_games')
@@ -110,6 +118,51 @@ def compute_pitch_stats(data, player_type, pitch_class_type, min_pitches=1):
   return pitch_stats
 
 
+def compute_pitcher_stats(data, min_ip='qualified'):
+  data = data.filter(pl.col('ballKind') != '-')
+  data = (
+      compute_team_games(data)
+      .with_columns(
+          pl.when(pl.col('half_inning').str.ends_with('1')).then('home_games').otherwise('visitor_games').first().over('pitId').alias('games'),
+          pl.col('inning_code').unique().len().over('pitId').alias('IP') # inaccurate
+      )
+  )
+
+  if min_ip == 'qualified':
+    data = data.with_columns((pl.col('IP') >= pl.col('games')).alias('qualified'))
+  else:
+    data = data.with_columns((pl.col('IP') >= min_ip).alias('qualified'))
+
+  pitcher_stats = (
+      data
+      .group_by('pitId', 'pitcher_team_name_short')
+      .agg(
+          pl.col('pitcher_name').first(),
+          (pl.when(pl.col('presult').str.contains('strikeout')).then(1).otherwise(0).sum() / pl.col('pa_code').unique().len()).alias('K%'),
+          (pl.when(pl.col('presult') == 'Walk').then(1).otherwise(0).sum() / pl.col('pa_code').unique().len()).alias('BB%'),
+          (pl.col('csw').sum() / pl.col('pitch').sum()).alias('CSW%'),
+          pl.col('whiff').sum().alias('Whiffs'),
+          pl.col('aux_bresult').struct.field('batType').drop_nulls().value_counts(normalize=True),
+          pl.first('qualified')
+      )
+      .explode('batType')
+      .unnest('batType')
+      .pivot(on='batType', values='proportion')
+      .fill_null(0)
+      .with_columns(
+          (pl.col('G') + pl.col('B')).alias('GB%'),
+          (pl.col('F') + pl.col('P')).alias('FB%'),
+          pl.col('L').alias('LD%'),
+      )
+      .drop('G', 'F', 'B', 'P', 'L', 'null')
+      .with_columns(
+          (pl.when(pl.col('qualified')).then(pl.col(stat)).rank(descending=(stat == 'BB%'))/pl.when(pl.col('qualified')).then(pl.col(stat)).count()).alias(f'{stat}_pctl')
+          for stat in ['CSW%', 'K%', 'BB%', 'GB%']
+      )
+  )
+  return pitcher_stats
+  
+
 def get_pitcher_stats(id, lr=None, game_kind=None, start_date=None, end_date=None, min_ip=1, min_pitches=1, pitch_class_type='specific'):
   source_data = data_df.filter(pl.col('ballKind_code') != '-')