Add logistic regression model and ML preprocessing

Introduces a new LogisticRegressionModel for fight outcome prediction, including a comprehensive ML preprocessing module (preprocess.py) for feature engineering and data preparation. Refactors model interfaces to accept fight dictionaries, updates the pipeline and main entrypoint for model selection and reporting, and improves data cleaning in the scraping and preprocessing steps. Also fixes fighter name formatting and adds pandas and scikit-learn as dependencies.

output/ufc_fighters.csv

@@ -374,7 +374,7 @@ Brad,Blackburn,Bad Brad,18,13,1,False,178,170,73.0,Orthodox,"May 25, 1977",http:
 Jason,Blackford,,3,3,0,False,,,,,,http://ufcstats.com/fighter-details/619d807fa54ae8f7,,,1500
 Sherrard,Blackledge,The Thriller,5,1,0,False,180,155,75.0,Orthodox,"Aug 16, 1993",http://ufcstats.com/fighter-details/0e5c79b3594ff0ad,75,155,1500
 Tom,Blackledge,,10,7,0,False,,205,,Orthodox,,http://ufcstats.com/fighter-details/2adb11835acd815b,,205,1477
-Da ftMon,Blackshear,The Monster,17,7,1,False,178,135,72.0,Switch,"Aug 12, 1994",http://ufcstats.com/fighter-details/da22387a0407a2dc,72,135,1500
+Da'Mon,Blackshear,The Monster,17,7,1,False,178,135,72.0,Switch,"Aug 12, 1994",http://ufcstats.com/fighter-details/da22387a0407a2dc,72,135,1500
 Chasen,Blair,Mestizo,6,3,0,False,178,155,71.0,Orthodox,"Oct 14, 1998",http://ufcstats.com/fighter-details/0f4a536507f33576,71,155,1500
 Erin,Blanchfield,Cold Blooded,13,2,0,False,163,125,66.0,Orthodox,"May 04, 1999",http://ufcstats.com/fighter-details/669970f7feba8ecd,66,125,1622
 David,Blanco,,2,0,0,False,170,145,,,,http://ufcstats.com/fighter-details/ebf298f8ac7e232b,,145,1500
@@ -831,7 +831,7 @@ Will,Currie,Drago,12,4,0,False,190,185,76.0,Switch,"Nov 12, 1998",http://ufcstat
 Chris,Curtis,Action-Man,31,12,0,False,178,170,75.0,Orthodox,"Jul 15, 1987",http://ufcstats.com/fighter-details/5442f1bc4b47eaf3,75,170,1529
 Ion,Cutelaba,The Hulk,19,11,1,False,185,205,75.0,Southpaw,"Dec 14, 1993",http://ufcstats.com/fighter-details/cd13728ae1151f46,75,205,1492
 Gleidson,Cutis,Poney,7,4,0,False,175,155,,Orthodox,"Feb 07, 1989",http://ufcstats.com/fighter-details/44a94bbde42246e4,,155,1500
-Sarah,D ftalelio,The Monster,11,7,0,False,170,145,,,"Dec 13, 1980",http://ufcstats.com/fighter-details/ac45450f75d14f16,,145,1500
+Sarah,D'alelio,The Monster,11,7,0,False,170,145,,,"Dec 13, 1980",http://ufcstats.com/fighter-details/ac45450f75d14f16,,145,1500
 Alex,Da Silva,Leko,21,4,0,False,173,155,73.0,Orthodox,"Feb 04, 1996",http://ufcstats.com/fighter-details/c3ded6f7155f9ea4,73,155,1466
 Paulo,Da Silva,PH,11,1,0,False,,125,,,"Jul 03, 2002",http://ufcstats.com/fighter-details/3fe8ad7e1594537a,,125,1500
 Radley,Da Silva,Snake Eyes,7,1,0,False,,145,,,"Jun 20, 1995",http://ufcstats.com/fighter-details/0d68c7bfdff1dc34,,145,1500
@@ -1925,7 +1925,7 @@ Tetsuji,Kato,,20,10,0,False,,155,,Orthodox,,http://ufcstats.com/fighter-details/
 Brad,Katona,Superman,16,5,0,False,168,135,64.0,Orthodox,"Dec 19, 1991",http://ufcstats.com/fighter-details/7b433309b0fd12aa,64,135,1471
 Calvin,Kattar,,23,9,0,False,180,145,72.0,Orthodox,"Mar 26, 1988",http://ufcstats.com/fighter-details/751de04455cfaac0,72,145,1530
 Sarah,Kaufman,,18,4,0,False,165,135,66.0,Orthodox,"Sep 20, 1985",http://ufcstats.com/fighter-details/36df8e119aec6175,66,135,1481
-Lone fter,Kavanagh,,9,0,0,False,163,125,67.0,Orthodox,"Jun 09, 1999",http://ufcstats.com/fighter-details/bb2c3c3a466224af,67,125,1500
+Lone'er,Kavanagh,,9,0,0,False,163,125,67.0,Orthodox,"Jun 09, 1999",http://ufcstats.com/fighter-details/bb2c3c3a466224af,67,125,1500
 Yusuke,Kawaguchi,,18,12,0,False,183,255,,,"Aug 14, 1980",http://ufcstats.com/fighter-details/fa2320781bfe4f49,,255,1500
 Canaan,Kawaihae,,6,2,0,False,183,145,71.0,Southpaw,"Aug 26, 1997",http://ufcstats.com/fighter-details/58d42b9e920b25fc,71,145,1500
 Tatsuya,Kawajiri,Crusher,36,11,2,False,170,145,69.0,Orthodox,"May 08, 1978",http://ufcstats.com/fighter-details/80d918336163b80c,69,145,1509
@@ -2432,7 +2432,7 @@ Nick,Maximov,,8,2,0,False,183,185,76.0,Southpaw,"Dec 23, 1997",http://ufcstats.c
 Elaina,Maxwell,Beef,7,4,0,False,173,145,,Orthodox,"Dec 16, 1978",http://ufcstats.com/fighter-details/a16ce18149021139,,145,1500
 Jack,May,The Outlaw,9,3,0,False,203,255,,Switch,"Apr 14, 1981",http://ufcstats.com/fighter-details/2c6e81426dd7573c,,255,1462
 Jeremy,May,,13,10,0,False,185,185,,,"Oct 23, 1986",http://ufcstats.com/fighter-details/c32fdfe75cda5b22,,185,1500
-Don ftTale,Mayes,Lord Kong,11,9,0,False,198,260,81.0,Orthodox,"Jan 16, 1992",http://ufcstats.com/fighter-details/1a9480fc288e55d7,81,260,1500
+Don'Tale,Mayes,Lord Kong,11,9,0,False,198,260,81.0,Orthodox,"Jan 16, 1992",http://ufcstats.com/fighter-details/1a9480fc288e55d7,81,260,1500
 Gray,Maynard,,13,7,1,False,175,155,70.0,Orthodox,"May 09, 1979",http://ufcstats.com/fighter-details/7d96bc577e5178b2,70,155,1544
 Brooke,Mayo,The Bully,0,1,0,False,170,125,,,,http://ufcstats.com/fighter-details/1b41c21d947d6f2f,,125,1500
 Gina,Mazany,Danger,7,6,0,False,168,125,68.0,Southpaw,"Aug 19, 1988",http://ufcstats.com/fighter-details/016a8d958883167c,68,125,1429
@@ -2824,16 +2824,16 @@ Said,Nurmagomedov,,18,4,0,False,173,135,70.0,Orthodox,"Apr 05, 1992",http://ufcs
 Umar,Nurmagomedov,,18,1,0,False,173,135,69.0,Orthodox,"Jan 03, 1996",http://ufcstats.com/fighter-details/2b6fc1c02736833d,69,135,1605
 Adilet,Nurmatov,Kok-Zhal,13,2,0,False,,155,,,"Aug 02, 1997",http://ufcstats.com/fighter-details/689502703bbfe5f2,,155,1500
 Kennedy,Nzechukwu,African Savage,14,5,0,False,196,236,83.0,Southpaw,"Jun 13, 1992",http://ufcstats.com/fighter-details/8667caa0451d245b,83,236,1550
-Jake,O ftBrien,Irish,15,4,0,False,190,205,76.0,Orthodox,"Sep 25, 1984",http://ufcstats.com/fighter-details/20bcc9966affb19c,76,205,1500
-TJ,O ftBrien,The Spider,20,8,0,False,188,155,,Orthodox,"Jan 01, 1987",http://ufcstats.com/fighter-details/d25b93992f285953,,155,1500
-Sean,O ftConnell,The Real OC,17,9,0,False,185,205,74.0,Orthodox,"Sep 02, 1983",http://ufcstats.com/fighter-details/cb52f9490c2dc069,74,205,1500
-Dan,O ftConnor,Johnny Irish,6,6,0,False,163,115,,,"Nov 17, 1982",http://ufcstats.com/fighter-details/69ea0119f6f0dfe0,,115,1500
-Sean,O ftHaire,,4,2,0,False,196,270,,Orthodox,"Feb 25, 1971",http://ufcstats.com/fighter-details/46effbd1135423c5,,270,1500
-Sean,O ftMalley,Suga,18,3,0,False,180,135,72.0,Switch,"Oct 24, 1994",http://ufcstats.com/fighter-details/b50a426a33da0012,72,135,1500
-Jeremiah,O ftNeal,,13,25,0,False,173,260,,,"Oct 25, 1977",http://ufcstats.com/fighter-details/338fda4ec7034c5d,,260,1500
-Chuck,O ftNeil,Cold Steel,17,9,0,False,188,170,,,"Sep 22, 1985",http://ufcstats.com/fighter-details/56bc9ccb609df534,,170,1500
-Casey,O ftNeill,King,10,2,0,False,168,125,69.0,Orthodox,"Oct 07, 1997",http://ufcstats.com/fighter-details/04835018f90b118c,69,125,1500
-Brendan,O ftReilly,Badger,6,3,0,False,170,170,69.0,Orthodox,"Jun 24, 1987",http://ufcstats.com/fighter-details/494b0bfdbac74502,69,170,1500
+Jake,O'Brien,Irish,15,4,0,False,190,205,76.0,Orthodox,"Sep 25, 1984",http://ufcstats.com/fighter-details/20bcc9966affb19c,76,205,1500
+TJ,O'Brien,The Spider,20,8,0,False,188,155,,Orthodox,"Jan 01, 1987",http://ufcstats.com/fighter-details/d25b93992f285953,,155,1500
+Sean,O'Connell,The Real OC,17,9,0,False,185,205,74.0,Orthodox,"Sep 02, 1983",http://ufcstats.com/fighter-details/cb52f9490c2dc069,74,205,1500
+Dan,O'Connor,Johnny Irish,6,6,0,False,163,115,,,"Nov 17, 1982",http://ufcstats.com/fighter-details/69ea0119f6f0dfe0,,115,1500
+Sean,O'Haire,,4,2,0,False,196,270,,Orthodox,"Feb 25, 1971",http://ufcstats.com/fighter-details/46effbd1135423c5,,270,1500
+Sean,O'Malley,Suga,18,3,0,False,180,135,72.0,Switch,"Oct 24, 1994",http://ufcstats.com/fighter-details/b50a426a33da0012,72,135,1500
+Jeremiah,O'Neal,,13,25,0,False,173,260,,,"Oct 25, 1977",http://ufcstats.com/fighter-details/338fda4ec7034c5d,,260,1500
+Chuck,O'Neil,Cold Steel,17,9,0,False,188,170,,,"Sep 22, 1985",http://ufcstats.com/fighter-details/56bc9ccb609df534,,170,1500
+Casey,O'Neill,King,10,2,0,False,168,125,69.0,Orthodox,"Oct 07, 1997",http://ufcstats.com/fighter-details/04835018f90b118c,69,125,1500
+Brendan,O'Reilly,Badger,6,3,0,False,170,170,69.0,Orthodox,"Jun 24, 1987",http://ufcstats.com/fighter-details/494b0bfdbac74502,69,170,1500
 Takahiro,Oba,,5,7,1,False,173,200,,Southpaw,,http://ufcstats.com/fighter-details/7139cd2ae4bf6a29,,200,1500
 Nobuhiro,Obiya,,12,5,1,False,173,145,,Orthodox,"Jan 15, 1981",http://ufcstats.com/fighter-details/6e3282d57d2467a0,,145,1500
 Jose,Ochoa,Kalzifer,8,1,0,False,170,125,67.0,Southpaw,"Dec 31, 2000",http://ufcstats.com/fighter-details/88be62d6c1e6dadb,67,125,1502
@@ -2905,7 +2905,7 @@ Craig,Oxley,,0,3,0,False,175,155,,,"Feb 14, 1973",http://ufcstats.com/fighter-de
 Shungo,Oyama,,14,19,0,False,180,185,,Southpaw,"Apr 11, 1974",http://ufcstats.com/fighter-details/47b7e4e60813b7b2,,185,1500
 Ren,Ozaki,,6,1,2,False,170,135,68.0,Orthodox,"Dec 15, 2001",http://ufcstats.com/fighter-details/8997ee20b6a43d76,68,135,1500
 Alptekin,Ozkilic,The Turkish Delight,9,5,0,False,165,125,65.0,Orthodox,"Mar 27, 1986",http://ufcstats.com/fighter-details/e18a19001a3f7c7d,65,125,1463
-Raquel,Pa ftaluhi,Rocky,6,5,0,False,170,135,,,,http://ufcstats.com/fighter-details/373be586f370d400,,135,1500
+Raquel,Pa'aluhi,Rocky,6,5,0,False,170,135,,,,http://ufcstats.com/fighter-details/373be586f370d400,,135,1500
 Nick,Pace,,8,3,0,False,170,135,68.0,Orthodox,"Apr 17, 1987",http://ufcstats.com/fighter-details/8cb76103cd8a1562,68,135,1479
 Angel,Pacheco,,7,3,0,False,173,135,70.0,Orthodox,"Jan 13, 1992",http://ufcstats.com/fighter-details/07797f10b9569cfc,70,135,1479
 Larissa,Pacheco,,10,2,0,False,170,135,,Orthodox,"Sep 07, 1994",http://ufcstats.com/fighter-details/16b89be2f5c16fba,,135,1462
@@ -3438,7 +3438,7 @@ Hugo,Sandoval,,2,3,0,False,,135,,,,http://ufcstats.com/fighter-details/9d51bcc28
 Joseph,Sandoval,,7,7,0,False,170,135,,Southpaw,"May 11, 1986",http://ufcstats.com/fighter-details/696002b59f09d73b,,135,1461
 Raul,Sandoval,,3,3,0,False,,130,,,,http://ufcstats.com/fighter-details/f9ad10f6a49e5452,,130,1500
 Chris,Sanford,,5,1,0,False,180,185,,Orthodox,"Mar 12, 1968",http://ufcstats.com/fighter-details/29f935654825331b,,185,1480
-Roldan,Sangcha ftan,The Executioner,4,2,0,False,163,125,,Orthodox,"Dec 04, 1990",http://ufcstats.com/fighter-details/57887765f831e228,,125,1500
+Roldan,Sangcha'an,The Executioner,4,2,0,False,163,125,,Orthodox,"Dec 04, 1990",http://ufcstats.com/fighter-details/57887765f831e228,,125,1500
 Martin,Sano,Spartan,4,3,1,False,180,170,75.0,Orthodox,"May 30, 1991",http://ufcstats.com/fighter-details/16a64f93f6678b7b,75,170,1481
 Yuhi,Sano,,0,4,0,False,180,243,,Orthodox,"Feb 02, 1965",http://ufcstats.com/fighter-details/4c12aa7ca246e7a4,,243,1500
 Jonathan,Santa Maria,,3,4,0,False,175,125,,,,http://ufcstats.com/fighter-details/3143e5daff9e5b71,,125,1500

requirements.txt

@@ -1,2 +1,4 @@
 requests
-beautifulsoup4 
+beautifulsoup4
+pandas
+scikit-learn 

src/predict/main.py

@@ -1,30 +1,37 @@
-from .models import EloBaselineModel
+import argparse
+from .models import EloBaselineModel, LogisticRegressionModel
 from .pipeline import PredictionPipeline
 
 def main():
     """
-    Sets up the models and runs the prediction pipeline.
-    This is where you can add new models to compare them.
+    Main entry point to run the prediction pipeline.
+    You can specify which models to run and the reporting format.
     """
-    print("--- Initializing Machine Learning Prediction Pipeline ---")
+    parser = argparse.ArgumentParser(description="UFC Fight Prediction Pipeline")
+    parser.add_argument(
+        '--report', 
+        type=str, 
+        default='detailed', 
+        choices=['detailed', 'summary'],
+        help="Type of report to generate: 'detailed' (file) or 'summary' (console)."
+    )
+    args = parser.parse_args()
 
-    # 1. Initialize the models you want to test
-    elo_model = EloBaselineModel()
-    
-    # Add other models here to compare them, e.g.:
-    # logistic_model = LogisticRegressionModel()
-    
-    # 2. Create a list of the models to evaluate
+    # --- Define Models to Run ---
+    # Instantiate all the models you want to evaluate here.
     models_to_run = [
-        elo_model,
-        # logistic_model
+        EloBaselineModel(),
+        LogisticRegressionModel(),
     ]
+    # --- End of Model Definition ---
 
-    # 3. Initialize and run the pipeline
     pipeline = PredictionPipeline(models=models_to_run)
     
-    # Set detailed_report=False for a summary, or True for a full detailed report
-    pipeline.run(detailed_report=True)
+    try:
+        pipeline.run(detailed_report=(args.report == 'detailed'))
+    except FileNotFoundError as e:
+        print(f"Error: {e}")
+        print("Please ensure the required data files exist. You may need to run the scraping and ELO analysis first.")
 
 if __name__ == '__main__':
     main() 

src/predict/models.py

@@ -2,6 +2,10 @@ from abc import ABC, abstractmethod
 import sys
 import os
 from ..analysis.elo import process_fights_for_elo, INITIAL_ELO
+import pandas as pd
+from sklearn.linear_model import LogisticRegression
+from ..config import FIGHTERS_CSV_PATH
+from .preprocess import preprocess_for_ml, _get_fighter_history_stats, _calculate_age
 
 class BaseModel(ABC):
     """
@@ -18,12 +22,11 @@ class BaseModel(ABC):
         pass
 
     @abstractmethod
-    def predict(self, fighter1_name, fighter2_name):
+    def predict(self, fight):
         """
         Predicts the winner of a single fight.
 
-        :param fighter1_name: The name of the first fighter.
-        :param fighter2_name: The name of the second fighter.
+        :param fight: A dictionary representing a single fight.
         :return: The name of the predicted winning fighter.
         """
         pass
@@ -33,24 +36,113 @@ class EloBaselineModel(BaseModel):
     A baseline prediction model that predicts the winner based on the higher ELO rating.
     """
     def __init__(self):
-        self.historical_elos = {}
+        self.fighters_df = None
 
     def train(self, train_fights):
         """
-        Calculates the ELO ratings for all fighters based on historical data.
-        These ratings are then stored to be used for predictions.
+        For the ELO baseline, 'training' simply consists of loading the fighter data
+        to access their ELO scores during prediction.
         """
-        print("Training ELO Baseline Model...")
-        self.historical_elos = process_fights_for_elo(train_fights)
-        print("ELO Model training complete.")
+        print("Training EloBaselineModel: Loading fighter ELO data...")
+        self.fighters_df = pd.read_csv(FIGHTERS_CSV_PATH)
+        self.fighters_df['full_name'] = self.fighters_df['first_name'] + ' ' + self.fighters_df['last_name']
+        self.fighters_df = self.fighters_df.drop_duplicates(subset=['full_name']).set_index('full_name')
 
-    def predict(self, fighter1_name, fighter2_name):
+    def predict(self, fight):
+        """Predicts the winner based on who has the higher ELO score."""
+        f1_name, f2_name = fight['fighter_1'], fight['fighter_2']
+        
+        try:
+            f1_elo = self.fighters_df.loc[f1_name, 'elo']
+            f2_elo = self.fighters_df.loc[f2_name, 'elo']
+            
+            return f1_name if f1_elo > f2_elo else f2_name
+        except KeyError as e:
+            # If a fighter isn't found, we can't make a prediction.
+            # Returning None or a default is a design choice.
+            print(f"Warning: Could not find ELO for fighter {e}. Skipping prediction.")
+            return None
+
+class LogisticRegressionModel(BaseModel):
+    """
+    A model that uses logistic regression to predict fight outcomes based on differential features.
+    """
+    def __init__(self):
+        self.model = LogisticRegression(solver='liblinear', random_state=42)
+        self.fighters_df = None
+        self.fighter_histories = {}
+
+    def train(self, train_fights):
+        """
+        Trains the logistic regression model by preprocessing the training data
+        and fitting the model.
+        """
+        print("Training LogisticRegressionModel...")
+        
+        # 1. Prepare data for prediction-time feature generation
+        self.fighters_df = pd.read_csv(FIGHTERS_CSV_PATH)
+        self.fighters_df['full_name'] = self.fighters_df['first_name'] + ' ' + self.fighters_df['last_name']
+        self.fighters_df = self.fighters_df.drop_duplicates(subset=['full_name']).set_index('full_name')
+        for col in ['height_cm', 'reach_in', 'elo']:
+            if col in self.fighters_df.columns:
+                self.fighters_df[col] = pd.to_numeric(self.fighters_df[col], errors='coerce')
+
+        # 2. Pre-calculate fighter histories for efficient lookup during prediction
+        train_fights_with_dates = []
+        for fight in train_fights:
+            fight['date_obj'] = pd.to_datetime(fight['event_date'])
+            train_fights_with_dates.append(fight)
+            
+        for fighter_name in self.fighters_df.index:
+            history = [f for f in train_fights_with_dates if fighter_name in (f['fighter_1'], f['fighter_2'])]
+            self.fighter_histories[fighter_name] = sorted(history, key=lambda x: x['date_obj'])
+
+        # 3. Preprocess training data and fit the model
+        X_train, y_train, _ = preprocess_for_ml(train_fights, FIGHTERS_CSV_PATH)
+        print(f"Fitting model on {X_train.shape[0]} samples...")
+        self.model.fit(X_train, y_train)
+        print("Model training complete.")
+
+    def predict(self, fight):
         """
-        Predicts the winner based on which fighter has the higher historical ELO.
-        If a fighter has no ELO rating, the default initial ELO is used.
+        Predicts the outcome of a single fight by generating its feature vector.
         """
-        elo1 = self.historical_elos.get(fighter1_name, INITIAL_ELO)
-        elo2 = self.historical_elos.get(fighter2_name, INITIAL_ELO)
+        f1_name, f2_name = fight['fighter_1'], fight['fighter_2']
+        fight_date = pd.to_datetime(fight['event_date'])
+
+        if f1_name not in self.fighters_df.index or f2_name not in self.fighters_df.index:
+            print(f"Warning: Fighter not found in data. Skipping prediction for {f1_name} vs {f2_name}")
+            return None
+
+        # 1. Get base stats
+        f1_stats, f2_stats = self.fighters_df.loc[f1_name], self.fighters_df.loc[f2_name]
+        if isinstance(f1_stats, pd.DataFrame): f1_stats = f1_stats.iloc[0]
+        if isinstance(f2_stats, pd.DataFrame): f2_stats = f2_stats.iloc[0]
+        
+        # 2. Get historical stats
+        f1_hist_stats = _get_fighter_history_stats(f1_name, fight_date, self.fighter_histories.get(f1_name, []), self.fighters_df)
+        f2_hist_stats = _get_fighter_history_stats(f2_name, fight_date, self.fighter_histories.get(f2_name, []), self.fighters_df)
+        
+        # 3. Create differential features
+        f1_age = _calculate_age(f1_stats.get('dob'), fight['event_date'])
+        f2_age = _calculate_age(f2_stats.get('dob'), fight['event_date'])
+
+        features = {
+            'elo_diff': f1_stats.get('elo', 1500) - f2_stats.get('elo', 1500),
+            'height_diff_cm': f1_stats.get('height_cm', 0) - f2_stats.get('height_cm', 0),
+            'reach_diff_in': f1_stats.get('reach_in', 0) - f2_stats.get('reach_in', 0),
+            'age_diff_years': (f1_age - f2_age) if f1_age and f2_age else 0,
+            'stance_is_different': 1 if f1_stats.get('stance') != f2_stats.get('stance') else 0,
+            'wins_last_5_diff': f1_hist_stats['wins_last_n'] - f2_hist_stats['wins_last_n'],
+            'avg_opp_elo_last_5_diff': f1_hist_stats['avg_opp_elo_last_n'] - f2_hist_stats['avg_opp_elo_last_n'],
+            'ko_percent_last_5_diff': f1_hist_stats['ko_percent_last_n'] - f2_hist_stats['ko_percent_last_n'],
+            'sig_str_landed_per_min_last_5_diff': f1_hist_stats['sig_str_landed_per_min_last_n'] - f2_hist_stats['sig_str_landed_per_min_last_n'],
+        }
+        
+        feature_vector = pd.DataFrame([features]).fillna(0)
+        
+        # 4. Predict
+        # The model predicts the probability of class '1' (a win for fighter_1)
+        prediction = self.model.predict(feature_vector)[0]
         
-        # Return the name of the fighter with the higher ELO
-        return fighter1_name if elo1 > elo2 else fighter2_name 
+        return f1_name if prediction == 1 else f2_name 

src/predict/pipeline.py

@@ -64,7 +64,8 @@ class PredictionPipeline:
                 f1_name, f2_name = fight['fighter_1'], fight['fighter_2']
                 actual_winner = fight['winner']
                 event_name = fight.get('event_name', 'Unknown Event')
-                predicted_winner = model.predict(f1_name, f2_name)
+                
+                predicted_winner = model.predict(fight)
                 
                 is_correct = (predicted_winner == actual_winner)
                 if is_correct:

src/predict/preprocess.py

@@ -0,0 +1,227 @@
+import pandas as pd
+import os
+import sys
+from datetime import datetime
+from ..config import FIGHTERS_CSV_PATH
+
+def _clean_numeric_column(series):
+    """A helper to clean string columns into numbers, handling errors."""
+    series_str = series.astype(str)
+    return pd.to_numeric(series_str.str.replace(r'[^0-9.]', '', regex=True), errors='coerce')
+
+def _calculate_age(dob_str, fight_date_str):
+    """Calculates age in years from a date of birth string and fight date string."""
+    if pd.isna(dob_str) or not dob_str:
+        return None
+    try:
+        dob = datetime.strptime(dob_str, '%b %d, %Y')
+        fight_date = datetime.strptime(fight_date_str, '%B %d, %Y')
+        return (fight_date - dob).days / 365.25
+    except (ValueError, TypeError):
+        return None
+
+def _parse_round_time_to_seconds(round_str, time_str):
+    """Converts fight duration from round and time to total seconds."""
+    try:
+        rounds = int(round_str)
+        minutes, seconds = map(int, time_str.split(':'))
+        # Assuming 5-minute rounds for calculation simplicity
+        return ((rounds - 1) * 5 * 60) + (minutes * 60) + seconds
+    except (ValueError, TypeError, AttributeError):
+        return 0
+
+def _parse_striking_stats(stat_str):
+    """Parses striking stats string like '10 of 20' into (landed, attempted)."""
+    try:
+        landed, attempted = map(int, stat_str.split(' of '))
+        return landed, attempted
+    except (ValueError, TypeError, AttributeError):
+        return 0, 0
+
+def _get_fighter_history_stats(fighter_name, current_fight_date, fighter_history, fighters_df, n=5):
+    """
+    Calculates performance statistics for a fighter based on their last n fights.
+    """
+    past_fights = [f for f in fighter_history if f['date_obj'] < current_fight_date]
+    last_n_fights = past_fights[-n:]
+
+    if not last_n_fights:
+        # Return a default dictionary with the correct keys for a fighter with no history
+        return {
+            'wins_last_n': 0,
+            'avg_opp_elo_last_n': 1500, # Assume average ELO for first opponent
+            'ko_percent_last_n': 0,
+            'sig_str_landed_per_min_last_n': 0,
+        }
+
+    stats = {
+        'wins': 0, 'ko_wins': 0, 'total_time_secs': 0,
+        'sig_str_landed': 0, 'opponent_elos': []
+    }
+
+    for fight in last_n_fights:
+        is_fighter_1 = (fight['fighter_1'] == fighter_name)
+        opponent_name = fight['fighter_2'] if is_fighter_1 else fight['fighter_1']
+
+        if fight['winner'] == fighter_name:
+            stats['wins'] += 1
+            if 'KO' in fight['method']:
+                stats['ko_wins'] += 1
+
+        if opponent_name in fighters_df.index:
+            opp_elo = fighters_df.loc[opponent_name, 'elo']
+            stats['opponent_elos'].append(opp_elo if pd.notna(opp_elo) else 1500)
+        
+        stats['total_time_secs'] += _parse_round_time_to_seconds(fight['round'], fight['time'])
+        
+        sig_str_stat = fight.get(f'f1_sig_str' if is_fighter_1 else 'f2_sig_str', '0 of 0')
+        landed, _ = _parse_striking_stats(sig_str_stat)
+        stats['sig_str_landed'] += landed
+
+    # Final calculations
+    avg_opp_elo = sum(stats['opponent_elos']) / len(stats['opponent_elos']) if stats['opponent_elos'] else 1500
+    
+    return {
+        'wins_last_n': stats['wins'],
+        'avg_opp_elo_last_n': avg_opp_elo,
+        'ko_percent_last_n': (stats['ko_wins'] / stats['wins']) if stats['wins'] > 0 else 0,
+        'sig_str_landed_per_min_last_n': (stats['sig_str_landed'] * 60 / stats['total_time_secs']) if stats['total_time_secs'] > 0 else 0,
+    }
+
+def preprocess_for_ml(fights_to_process, fighters_csv_path):
+    """
+    Transforms raw fight and fighter data into a feature matrix (X) and target vector (y)
+    suitable for a binary classification machine learning model.
+
+    Args:
+        fights_to_process (list of dict): The list of fights to process.
+        fighters_csv_path (str): Path to the CSV file with all fighter stats.
+
+    Returns:
+        pd.DataFrame: Feature matrix X.
+        pd.Series: Target vector y.
+        pd.DataFrame: Metadata DataFrame.
+    """
+    if not os.path.exists(fighters_csv_path):
+        raise FileNotFoundError(f"Fighters data not found at '{fighters_csv_path}'.")
+
+    fighters_df = pd.read_csv(fighters_csv_path)
+    
+    # 1. Prepare fighters data for merging
+    fighters_prepared = fighters_df.copy()
+    fighters_prepared['full_name'] = fighters_prepared['first_name'] + ' ' + fighters_prepared['last_name']
+    
+    # Handle duplicate fighter names by keeping the first entry
+    fighters_prepared = fighters_prepared.drop_duplicates(subset=['full_name'], keep='first')
+    fighters_prepared = fighters_prepared.set_index('full_name')
+
+    for col in ['height_cm', 'reach_in', 'elo']:
+        if col in fighters_prepared.columns:
+            fighters_prepared[col] = _clean_numeric_column(fighters_prepared[col])
+
+    # 2. Pre-calculate fighter histories to speed up lookups
+    # And convert date strings to datetime objects once
+    for fight in fights_to_process:
+        fight['date_obj'] = datetime.strptime(fight['event_date'], '%B %d, %Y')
+    
+    fighter_histories = {}
+    for fighter_name in fighters_prepared.index:
+        history = [f for f in fights_to_process if fighter_name in (f['fighter_1'], f['fighter_2'])]
+        fighter_histories[fighter_name] = sorted(history, key=lambda x: x['date_obj'])
+
+    # 3. Process fights to create features and targets
+    feature_list = []
+    target_list = []
+    metadata_list = []
+
+    for fight in fights_to_process:
+        # Per the dataset's design, fighter_1 is always the winner.
+        f1_name, f2_name = fight['fighter_1'], fight['fighter_2']
+
+        if f1_name not in fighters_prepared.index or f2_name not in fighters_prepared.index:
+            continue
+
+        f1_stats, f2_stats = fighters_prepared.loc[f1_name], fighters_prepared.loc[f2_name]
+        
+        if isinstance(f1_stats, pd.DataFrame): f1_stats = f1_stats.iloc[0]
+        if isinstance(f2_stats, pd.DataFrame): f2_stats = f2_stats.iloc[0]
+
+        # Calculate ages for both fighters
+        f1_age = _calculate_age(f1_stats.get('dob'), fight['event_date'])
+        f2_age = _calculate_age(f2_stats.get('dob'), fight['event_date'])
+
+        # Get historical stats for both fighters
+        f1_hist_stats = _get_fighter_history_stats(f1_name, fight['date_obj'], fighter_histories.get(f1_name, []), fighters_prepared)
+        f2_hist_stats = _get_fighter_history_stats(f2_name, fight['date_obj'], fighter_histories.get(f2_name, []), fighters_prepared)
+        
+        # --- Create two training examples from each fight for a balanced dataset ---
+
+        # 1. The "Win" case: (fighter_1 - fighter_2)
+        features_win = {
+            # Original diffs
+            'elo_diff': f1_stats.get('elo', 1500) - f2_stats.get('elo', 1500),
+            'height_diff_cm': f1_stats.get('height_cm', 0) - f2_stats.get('height_cm', 0),
+            'reach_diff_in': f1_stats.get('reach_in', 0) - f2_stats.get('reach_in', 0),
+            'age_diff_years': (f1_age - f2_age) if f1_age and f2_age else 0,
+            'stance_is_different': 1 if f1_stats.get('stance') != f2_stats.get('stance') else 0,
+            # New historical diffs
+            'wins_last_5_diff': f1_hist_stats['wins_last_n'] - f2_hist_stats['wins_last_n'],
+            'avg_opp_elo_last_5_diff': f1_hist_stats['avg_opp_elo_last_n'] - f2_hist_stats['avg_opp_elo_last_n'],
+            'ko_percent_last_5_diff': f1_hist_stats['ko_percent_last_n'] - f2_hist_stats['ko_percent_last_n'],
+            'sig_str_landed_per_min_last_5_diff': f1_hist_stats['sig_str_landed_per_min_last_n'] - f2_hist_stats['sig_str_landed_per_min_last_n'],
+        }
+        feature_list.append(features_win)
+        target_list.append(1)  # 1 represents a win
+
+        # 2. The "Loss" case: (fighter_2 - fighter_1)
+        # We invert the differences for the losing case.
+        features_loss = {key: -value for key, value in features_win.items()}
+        # Stance difference is symmetric; it doesn't get inverted.
+        features_loss['stance_is_different'] = features_win['stance_is_different']
+        
+        feature_list.append(features_loss)
+        target_list.append(0)  # 0 represents a loss
+
+        # Add metadata for both generated samples
+        # The 'winner' and 'loser' are consistent with the original data structure
+        metadata_list.append({
+            'winner': f1_name, 'loser': f2_name, 'event_date': fight['event_date']
+        })
+        metadata_list.append({
+            'winner': f1_name, 'loser': f2_name, 'event_date': fight['event_date']
+        })
+
+    X = pd.DataFrame(feature_list).fillna(0)
+    y = pd.Series(target_list, name='winner')
+    metadata = pd.DataFrame(metadata_list)
+
+    print(f"Preprocessing complete. Generated {X.shape[0]} samples with {X.shape[1]} features.")
+    return X, y, metadata
+
+if __name__ == '__main__':
+    from .pipeline import PredictionPipeline
+    
+    print("--- Running Preprocessing Example ---")
+    
+    pipeline = PredictionPipeline(models=[])
+    try:
+        pipeline._load_and_split_data()
+        if pipeline.train_fights:
+            X_train, y_train, metadata_train = preprocess_for_ml(pipeline.train_fights, FIGHTERS_CSV_PATH)
+            print("\nTraining Data Shape:")
+            print("X_train:", X_train.shape)
+            print("y_train:", y_train.shape)
+            print("metadata_train:", metadata_train.shape)
+
+            print("\nLast 5 rows of X_train (showing populated historical features):")
+            print(X_train.tail())
+            
+            print("\nTarget distribution (0=Loss, 1=Win):")
+            print(y_train.value_counts())
+            
+            print("\nMetadata for last 5 rows:")
+            print(metadata_train.tail())
+
+    except FileNotFoundError as e:
+        print(e)
+        print("Please run the scraping pipeline first ('python -m src.scrape.main').")

src/scrape/main.py

@@ -8,37 +8,29 @@ from .. import config
 
 def main():
     """
-    Main pipeline to scrape UFC data and convert it to CSV.
+    Main function to run the complete scraping and preprocessing pipeline.
     """
     # Ensure the output directory exists
     if not os.path.exists(config.OUTPUT_DIR):
         os.makedirs(config.OUTPUT_DIR)
         print(f"Created directory: {config.OUTPUT_DIR}")
 
-    # --- Step 1: Scrape Events and Fights ---
-    print("\n--- Starting Events and Fights Scraping ---")
-    all_events_data = scrape_all_events()
-    with open(config.EVENTS_JSON_PATH, 'w') as f:
-        json.dump(all_events_data, f, indent=4)
-    print(f"Scraping for events complete. Data saved to {config.EVENTS_JSON_PATH}")
+    # --- Step 1: Scrape all data from the website ---
+    # This will generate fighters.json and events.json
+    scrape_all_fighters()
+    scrape_all_events()
 
-    # --- Step 2: Scrape Fighters ---
-    print("\n--- Starting Fighters Scraping ---")
-    all_fighters_data = scrape_all_fighters()
-    with open(config.FIGHTERS_JSON_PATH, 'w') as f:
-        json.dump(all_fighters_data, f, indent=4)
-    print(f"Scraping for fighters complete. Data saved to {config.FIGHTERS_JSON_PATH}")
-
-    # --- Step 3: Convert JSON to CSV ---
-    print("\n--- Converting all JSON files to CSV ---")
+    # --- Step 2: Convert the scraped JSON data to CSV format ---
+    # This will generate fighters.csv and fights.csv
     json_to_csv(config.EVENTS_JSON_PATH, config.FIGHTS_CSV_PATH)
     fighters_json_to_csv(config.FIGHTERS_JSON_PATH, config.FIGHTERS_CSV_PATH)
 
-    # --- Step 4: Preprocess CSV data ---
-    print("\n--- Preprocessing fighter data (converting height to cm) ---")
-    preprocess_fighters_csv(config.FIGHTERS_CSV_PATH)
+    # --- Step 3: Run post-processing on the generated CSV files ---
+    # This cleans names, converts height, etc.
+    print("\n--- Running post-scraping preprocessing ---")
+    preprocess_fighters_csv()
 
-    # --- Step 5: Clean up temporary JSON files ---
+    # --- Step 4: Clean up temporary JSON files ---
     print("\n--- Deleting temporary JSON files ---")
     try:
         if os.path.exists(config.EVENTS_JSON_PATH):
@@ -50,7 +42,7 @@ def main():
     except OSError as e:
         print(f"Error deleting JSON files: {e}")
 
-    print("\n--- Pipeline Finished ---")
+    print("\n\n--- Scraping and Preprocessing Pipeline Finished ---")
 
 if __name__ == '__main__':
     main()

src/scrape/preprocess.py

@@ -26,7 +26,7 @@ def convert_height_to_cm(height_str):
 
 def preprocess_fighters_csv(file_path=config.FIGHTERS_CSV_PATH):
     """
-    Reads the fighters CSV, converts height to cm, renames the column,
+    Reads the fighters CSV, cleans names, converts height to cm, 
     and saves the changes back to the same file.
     """
     if not os.path.exists(file_path):
@@ -46,18 +46,24 @@ def preprocess_fighters_csv(file_path=config.FIGHTERS_CSV_PATH):
             headers = reader.fieldnames
             rows = list(reader)
 
-        # Check if there's a 'height' column to process
-        if 'height' not in headers:
-            print("No 'height' column found. Nothing to do.")
-            return
-
+        # --- Data Cleaning and Processing ---
+        
+        name_cleaned_count = 0
         # Process the rows in memory
         for row in rows:
-            # Create a new key for the converted height and remove the old one
-            row['height_cm'] = convert_height_to_cm(row.pop('height', ''))
+            # Clean fighter names (e.g., "O ftMalley" -> "O'Malley")
+            for col in ['first_name', 'last_name']:
+                if col in row and ' ft' in row[col]:
+                    row[col] = row[col].replace(' ft', "'")
+                    name_cleaned_count += 1
+
+            # Convert height to cm and remove the old column
+            if 'height' in row:
+                row['height_cm'] = convert_height_to_cm(row.pop('height'))
 
-        # Update the header name
-        headers[headers.index('height')] = 'height_cm'
+        # Update the header name if 'height' was present
+        if 'height' in headers:
+            headers[headers.index('height')] = 'height_cm'
 
         # Write the modified data back to the same file, overwriting it
         with open(file_path, 'w', newline='', encoding='utf-8') as csv_file:
@@ -66,7 +72,10 @@ def preprocess_fighters_csv(file_path=config.FIGHTERS_CSV_PATH):
             writer.writerows(rows)
 
         print(f"Successfully processed file: {file_path}")
-        print("Converted 'height' column to centimeters and renamed it to 'height_cm'.")
+        if name_cleaned_count > 0:
+            print(f"Cleaned {name_cleaned_count} instances of ' ft' in fighter names.")
+        if 'height_cm' in headers:
+            print("Converted 'height' column to centimeters and renamed it to 'height_cm'.")
 
     except Exception as e:
         print(f"An error occurred: {e}")