adding invalid markets graphs

app.py

@@ -25,6 +25,14 @@ from tabs.tool_accuracy import (
     plot_tools_accuracy_graph,
     plot_tools_weighted_accuracy_graph,
 )
+
+from tabs.invalid_markets import (
+    plot_daily_dist_invalid_trades,
+    plot_ratio_invalid_trades_per_market,
+    plot_top_invalid_markets,
+    plot_daily_nr_invalid_markets,
+)
+
 from tabs.error import (
     get_error_data,
     get_error_data_overall,
@@ -86,11 +94,18 @@ def get_last_one_month_data():
 
 def get_all_data():
     """
-    Get all data from the tools.parquet, tools_accuracy and all_trades_profitability.parquet files
+    Get all data from the tools.parquet, tools_accuracy and trades parquet files
     """
     logger.info("Getting all data")
     con = duckdb.connect(":memory:")
 
+    # Query to fetch invalid trades data
+    query4 = f"""
+    SELECT *
+    FROM read_parquet('./data/invalid_trades.parquet')
+    """
+    df4 = con.execute(query4).fetchdf()
+
     # Query to fetch tools accuracy data
     query3 = f"""
     SELECT *
@@ -115,14 +130,14 @@ def get_all_data():
 
     con.close()
 
-    return df1, df2, df3
+    return df1, df2, df3, df4
 
 
 def prepare_data():
     """
     Prepare the data for the dashboard
     """
-    tools_df, trades_df, tools_accuracy_info = get_all_data()
+    tools_df, trades_df, tools_accuracy_info, invalid_trades = get_all_data()
 
     tools_df["request_time"] = pd.to_datetime(tools_df["request_time"])
     trades_df["creation_timestamp"] = pd.to_datetime(trades_df["creation_timestamp"])
@@ -133,10 +148,15 @@ def prepare_data():
     print("weighted accuracy info")
     print(tools_accuracy_info.head())
 
-    return tools_df, trades_df, tools_accuracy_info
+    invalid_trades["creation_timestamp"] = pd.to_datetime(
+        invalid_trades["creation_timestamp"]
+    )
+    invalid_trades["creation_date"] = invalid_trades["creation_timestamp"].dt.date
+
+    return tools_df, trades_df, tools_accuracy_info, invalid_trades
 
 
-tools_df, trades_df, tools_accuracy_info = prepare_data()
+tools_df, trades_df, tools_accuracy_info, invalid_trades = prepare_data()
 
 
 demo = gr.Blocks()
@@ -279,6 +299,27 @@ with demo:
             with gr.Row():
                 plot_tools_weighted_accuracy_graph(tools_accuracy_info)
 
+        with gr.TabItem("⛔ Invalid Markets Dashboard"):
+            with gr.Row():
+                gr.Markdown("# Daily distribution of invalid trades")
+            with gr.Row():
+                plot_daily_dist_invalid_trades(invalid_trades)
+
+            with gr.Row():
+                gr.Markdown("# Ratio of invalid trades per market")
+            with gr.Row():
+                plot_ratio_invalid_trades_per_market(invalid_trades)
+
+            with gr.Row():
+                gr.Markdown("# Top markets with invalid trades")
+            with gr.Row():
+                plot_top_invalid_markets(invalid_trades)
+
+            with gr.Row():
+                gr.Markdown("# Daily distribution of invalid markets")
+            with gr.Row():
+                plot_daily_nr_invalid_markets(invalid_trades)
+
         with gr.TabItem("🏥 Tool Error Dashboard"):
             with gr.Row():
                 gr.Markdown("# All tools errors")

data/fpmmTrades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1bdad279707286ea22f963e56e3aa9ba38341cbc5e5bce6f2c9384c95b949bf6
-size 5618058
+oid sha256:33d2a861848b7f5cd5ccd562355a16cc1ac6ee7ed41ae910d3e837290356b89c
+size 1372727

tabs/invalid_markets.py

@@ -0,0 +1,59 @@
+import pandas as pd
+import gradio as gr
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+
+def plot_daily_dist_invalid_trades(invalid_trades: pd.DataFrame):
+    """Function to paint the distribution of daily invalid trades, no matter which market"""
+    plot = sns.histplot(data=invalid_trades, x="creation_date", kde=True)
+    plt.xticks(rotation=45, ha="right")
+    plt.xlabel("Creation date")
+    plt.ylabel("Daily number of invalid trades")
+    plt.title("Distribution of daily invalid trades over time")
+    return gr.Plot(value=plot.get_figure())
+
+
+def plot_daily_nr_invalid_markets(invalid_trades: pd.DataFrame):
+    """Function to paint the number of invalid markets over time"""
+    daily_invalid_markets = (
+        invalid_trades.groupby("creation_date")
+        .agg(trades_count=("title", "count"), nr_markets=("title", "nunique"))
+        .reset_index()
+    )
+    sns.set_theme(palette="viridis")
+    plot = sns.lineplot(data=daily_invalid_markets, x="creation_date", y="nr_markets")
+    plt.xticks(rotation=45, ha="right")
+    plt.xlabel("Creation date")
+    plt.ylabel("Daily number of invalid markets")
+    plt.title("Evolution of daily invalid markets over time")
+    return gr.Plot(value=plot.get_figure())
+
+
+def plot_ratio_invalid_trades_per_market(invalid_trades: pd.DataFrame):
+    """Function to paint the number of invalid trades that the same market accummulates"""
+    cat = invalid_trades["title"]
+    codes, uniques = pd.factorize(cat)
+
+    # add the IDs as a new column to the original dataframe
+    invalid_trades["title_id"] = codes
+    plot = sns.displot(invalid_trades, x="title_id")
+    plt.xlabel("market id")
+    plt.ylabel("Total number of invalid trades by market")
+    plt.title("Distribution of invalid trades per market")
+    return gr.Plot(value=plot.get_figure())
+
+
+def plot_top_invalid_markets(invalid_trades: pd.DataFrame):
+    """Function to paint the top markets with the highest number of invalid trades"""
+    top_invalid_markets = invalid_trades.title.value_counts().reset_index()
+    top_invalid_markets.rename(columns={"count": "nr_invalid_trades"}, inplace=True)
+    plt.figure(figsize=(25, 10))
+    plot = sns.barplot(
+        top_invalid_markets,
+        x="nr_invalid_trades",
+        y="title",
+        hue="title",
+        dodge=False,
+    )
+    return gr.Plot(value=plot.get_figure())