Multiple Dataset Versions

app.py

@@ -3,11 +3,12 @@ import pandas as pd
 import numpy as np
 from bokeh.plotting import figure
 from bokeh.models import ColumnDataSource, DataTable, TableColumn, CustomJS, Select, Button
-from bokeh.layouts import row, column
-from bokeh.palettes import Reds9, Blues9
+from bokeh.layouts import column
+from bokeh.palettes import Reds9, Blues9, Oranges9, Purples9
 from sklearn.decomposition import PCA
 from sklearn.manifold import TSNE
 import io
+import ot
 
 TOOLTIPS = """
 <div>
@@ -30,20 +31,31 @@ def config_style():
     """, unsafe_allow_html=True)
     st.markdown('<h1 class="main-title">Merit Embeddings 🎒📃🏆</h1>', unsafe_allow_html=True)
 
-# Modificamos load_embeddings para aceptar el modelo a cargar
+# Carga los datos y asigna versiones de forma uniforme
 def load_embeddings(model):
     if model == "Donut":
         df_real = pd.read_csv("data/donut_de_Rodrigo_merit_secret_all_embeddings.csv")
-        df_es_digital_seq = pd.read_csv("data/donut_de_Rodrigo_merit_es-digital-seq_embeddings.csv")
+        df_seq  = pd.read_csv("data/donut_de_Rodrigo_merit_es-digital-seq_embeddings.csv")
+        df_line = pd.read_csv("data/idefics2_de_Rodrigo_merit_es-digital-seq_embeddings.csv")
+        df_real["version"] = "real"
+        df_seq["version"] = "synthetic"
+        df_line["version"] = "synthetic"
+        # Usamos un identificador en la columna 'source' para diferenciarlos
+        df_seq["source"] = "es-digital-seq"
+        df_line["source"] = "es-digital-line-degradation-seq"
+        return {"real": df_real, "synthetic": pd.concat([df_seq, df_line], ignore_index=True)}
     elif model == "Idefics2":
         df_real = pd.read_csv("data/idefics2_de_Rodrigo_merit_secret_britanico_embeddings.csv")
-        df_es_digital_seq = pd.read_csv("data/idefics2_de_Rodrigo_merit_es-digital-seq_embeddings.csv")
+        df_seq  = pd.read_csv("data/idefics2_de_Rodrigo_merit_es-digital-seq_embeddings.csv")
+        df_real["version"] = "real"
+        df_seq["version"] = "synthetic"
+        df_seq["source"] = "es-digital-seq"
+        return {"real": df_real, "synthetic": df_seq}
     else:
         st.error("Modelo no reconocido")
         return None
-    return {"real": df_real, "es-digital-seq": df_es_digital_seq}
 
-# Funciones auxiliares (idénticas a las de tu código)
+# Selección de reducción dimensional
 def reducer_selector(df_combined, embedding_cols):
     reduction_method = st.selectbox("Select Dimensionality Reduction Method:", options=["PCA", "t-SNE"])
     all_embeddings = df_combined[embedding_cols].values
@@ -53,7 +65,8 @@ def reducer_selector(df_combined, embedding_cols):
         reducer = TSNE(n_components=2, random_state=42, perplexity=30, learning_rate=200)
     return reducer.fit_transform(all_embeddings)
 
-def add_dataset_to_fig(fig, df, selected_labels, marker, color_mapping):
+# Función genérica para agregar datos al gráfico
+def add_dataset_to_fig(fig, df, selected_labels, marker, color_mapping, group_label):
     renderers = {}
     for label in selected_labels:
         subset = df[df['label'] == label]
@@ -63,112 +76,153 @@ def add_dataset_to_fig(fig, df, selected_labels, marker, color_mapping):
             x=subset['x'],
             y=subset['y'],
             label=subset['label'],
-            img=subset['img']
+            img=subset.get('img', "")
         ))
         color = color_mapping[label]
+        # Se añade el identificador de la fuente en la leyenda
+        legend_label = f"{label} ({group_label})"
         if marker == "circle":
             r = fig.circle('x', 'y', size=10, source=source,
                            fill_color=color, line_color=color,
-                           legend_label=f"{label} (Real)")
+                           legend_label=legend_label)
         elif marker == "square":
-            r = fig.square('x', 'y', size=6, source=source,
+            r = fig.square('x', 'y', size=10, source=source,
                            fill_color=color, line_color=color,
-                           legend_label=f"{label} (Synthetic)")
-        renderers[label] = r
+                           legend_label=legend_label)
+        elif marker == "triangle":
+            r = fig.triangle('x', 'y', size=12, source=source,
+                             fill_color=color, line_color=color,
+                             legend_label=legend_label)
+        renderers[label + f" ({group_label})"] = r
     return renderers
 
-def get_color_maps(selected_subsets: dict):
-    num_real = len(selected_subsets["real"])
+# Asigna paletas de colores de forma genérica para cada grupo (real y para cada fuente sintética)
+def get_color_maps(unique_subsets):
+    color_map = {}
+    # Real
+    num_real = len(unique_subsets["real"])
     red_palette = Reds9[:num_real] if num_real <= 9 else (Reds9 * ((num_real // 9) + 1))[:num_real]
-    color_mapping_real = {label: red_palette[i] for i, label in enumerate(sorted(selected_subsets["real"]))}
+    color_map["real"] = {label: red_palette[i] for i, label in enumerate(sorted(unique_subsets["real"]))}
     
-    num_es = len(selected_subsets["es-digital-seq"])
-    blue_palette = Blues9[:num_es] if num_es <= 9 else (Blues9 * ((num_es // 9) + 1))[:num_es]
-    color_mapping_es = {label: blue_palette[i] for i, label in enumerate(sorted(selected_subsets["es-digital-seq"]))}
-    
-    return {"real": color_mapping_real, "es-digital-seq": color_mapping_es}
+    # Synthetic: vamos a separar por fuente (source) basándonos en la lista completa de etiquetas
+    # Suponemos que en la columna "source" se encuentran los identificadores
+    synthetic_labels = sorted(unique_subsets["synthetic"])
+    # Aquí usamos una sola paleta para todos, pero se podría distinguir según la fuente si se quiere
+    blue_palette = Blues9[:len(synthetic_labels)] if len(synthetic_labels) <= 9 else (Blues9 * ((len(synthetic_labels) // 9) + 1))[:len(synthetic_labels)]
+    color_map["synthetic"] = {label: blue_palette[i] for i, label in enumerate(synthetic_labels)}
+    return color_map
 
+# Separa los datos reducidos en "real" y "synthetic" y extrae los subsets (clusters)
 def split_versions(df_combined, reduced):
     df_combined['x'] = reduced[:, 0]
     df_combined['y'] = reduced[:, 1]
     df_real = df_combined[df_combined["version"] == "real"].copy()
-    df_es = df_combined[df_combined["version"] == "es_digital_seq"].copy()
+    df_synth = df_combined[df_combined["version"] == "synthetic"].copy()
+    # Extraemos los clusters (subset) usando la columna 'label'
     unique_real = sorted(df_real['label'].unique().tolist())
-    unique_es = sorted(df_es['label'].unique().tolist())
-    return {"real": df_real, "es-digital-seq": df_es}, {"real": unique_real, "es-digital-seq": unique_es}
+    unique_synth = sorted(df_synth['label'].unique().tolist())
+    df_dict = {"real": df_real, "synthetic": df_synth}
+    unique_subsets = {"real": unique_real, "synthetic": unique_synth}
+    return df_dict, unique_subsets
 
-def create_figure(dfs_reduced, selected_subsets: dict, color_maps: dict):
+# Crea el gráfico; se tratan de forma uniforme ambos conjuntos sintéticos
+def create_figure(dfs, unique_subsets, color_maps):
     fig = figure(width=400, height=400, tooltips=TOOLTIPS, title="")
-    real_renderers = add_dataset_to_fig(fig, dfs_reduced["real"], selected_subsets["real"],
-                                        marker="circle", color_mapping=color_maps["real"])
-    synthetic_renderers = add_dataset_to_fig(fig, dfs_reduced["es-digital-seq"], selected_subsets["es-digital-seq"],
-                                             marker="square", color_mapping=color_maps["es-digital-seq"])
+    real_renderers = add_dataset_to_fig(fig, dfs["real"], unique_subsets["real"],
+                                        marker="circle", color_mapping=color_maps["real"],
+                                        group_label="Real")
+    # Aquí separamos los puntos sintéticos según su fuente para asignar diferentes marcadores
+    synth_df = dfs["synthetic"]
+    # Dividimos por 'source'
+    df_seq = synth_df[synth_df["source"] == "es-digital-seq"]
+    df_line = synth_df[synth_df["source"] == "es-digital-line-degradation-seq"]
+    
+    # Extraemos los clusters para cada fuente (si existen)
+    unique_seq = sorted(df_seq['label'].unique().tolist())
+    unique_line = sorted(df_line['label'].unique().tolist())
+    
+    seq_renderers = add_dataset_to_fig(fig, df_seq, unique_seq,
+                                       marker="square", color_mapping=color_maps["synthetic"],
+                                       group_label="es-digital-seq")
+    line_renderers = add_dataset_to_fig(fig, df_line, unique_line,
+                                        marker="triangle", color_mapping=color_maps["synthetic"],
+                                        group_label="es-digital-line-degradation-seq")
+    # Combina ambos renderers sintéticos
+    synthetic_renderers = {**seq_renderers, **line_renderers}
+    
     fig.legend.location = "top_right"
     fig.legend.click_policy = "hide"
     return fig, real_renderers, synthetic_renderers
 
-def calculate_cluster_centers(df: pd.DataFrame, selected_labels: list) -> dict:
+# Calcula los centros de cada cluster (por grupo)
+def calculate_cluster_centers(df, labels):
     centers = {}
-    for label in selected_labels:
+    for label in labels:
         subset = df[df['label'] == label]
         if not subset.empty:
             centers[label] = (subset['x'].mean(), subset['y'].mean())
     return centers
 
-def compute_distances(centers_es: dict, centers_real: dict) -> pd.DataFrame:
+# Calcula la distancia Wasserstein de cada subset sintético respecto a cada cluster real (por cluster y global)
+def compute_wasserstein_distances_all_synthetics(df_synth, df_real, labels_real):
     distances = {}
-    for es_label, (x_es, y_es) in centers_es.items():
-        distances[es_label] = {}
-        for real_label, (x_real, y_real) in centers_real.items():
-            distances[es_label][real_label] = np.sqrt((x_es - x_real)**2 + (y_es - y_real)**2)
+    # Para cada cluster en el conjunto sintético (la tabla mostrará todas las etiquetas)
+    synth_labels = sorted(df_synth['label'].unique().tolist())
+    for label in synth_labels:
+        key = f"{label}"
+        distances[key] = {}
+        cluster = df_synth[df_synth['label'] == label][['x','y']].values
+        n = cluster.shape[0]
+        weights = np.ones(n) / n
+        for real_label in labels_real:
+            cluster_real = df_real[df_real['label'] == real_label][['x','y']].values
+            m = cluster_real.shape[0]
+            weights_real = np.ones(m) / m
+            M = ot.dist(cluster, cluster_real, metric='euclidean')
+            distances[key][real_label] = ot.emd2(weights, weights_real, M)
+    # Distancia global del conjunto sintético a cada cluster real
+    key = "Global synthetic"
+    distances[key] = {}
+    global_synth = df_synth[['x','y']].values
+    n_global = global_synth.shape[0]
+    weights_global = np.ones(n_global) / n_global
+    for real_label in labels_real:
+        cluster_real = df_real[df_real['label'] == real_label][['x','y']].values
+        m = cluster_real.shape[0]
+        weights_real = np.ones(m) / m
+        M = ot.dist(global_synth, cluster_real, metric='euclidean')
+        distances[key][real_label] = ot.emd2(weights_global, weights_real, M)
     return pd.DataFrame(distances).T
 
 def create_table(df_distances):
     df_table = df_distances.copy()
     df_table.reset_index(inplace=True)
     df_table.rename(columns={'index': 'Synthetic'}, inplace=True)
-    
-    # Calcular las filas de medias, máximos y mínimos para cada columna numérica
     min_row = {"Synthetic": "Min."}
     mean_row = {"Synthetic": "Mean"}
     max_row = {"Synthetic": "Max."}
-    
     for col in df_table.columns:
         if col != "Synthetic":
             min_row[col] = df_table[col].min()
             mean_row[col] = df_table[col].mean()
             max_row[col] = df_table[col].max()
-    
-    # Agregar las filas de medias, máximos y mínimos al final del DataFrame
     df_table = pd.concat([df_table, pd.DataFrame([min_row, mean_row, max_row])], ignore_index=True)
-
     source_table = ColumnDataSource(df_table)
     columns = [TableColumn(field='Synthetic', title='Synthetic')]
     for col in df_table.columns:
         if col != 'Synthetic':
             columns.append(TableColumn(field=col, title=col))
-    
-    row_height = 28
-    header_height = 30
-    total_height = header_height + len(df_table) * row_height
-
+    total_height = 30 + len(df_table)*28
     data_table = DataTable(source=source_table, columns=columns, sizing_mode='stretch_width', height=total_height)
     return data_table, df_table, source_table
 
-
-
-# Función que ejecuta todo el proceso para un modelo determinado
 def run_model(model_name):
     embeddings = load_embeddings(model_name)
     if embeddings is None:
         return
-
-    # Asignamos la versión para distinguir en el split
-    embeddings["real"]["version"] = "real"
-    embeddings["es-digital-seq"]["version"] = "es_digital_seq"
     embedding_cols = [col for col in embeddings["real"].columns if col.startswith("dim_")]
-    df_combined = pd.concat([embeddings["real"], embeddings["es-digital-seq"]], ignore_index=True)
-    
+    # Combina todos los DataFrames
+    df_combined = pd.concat(list(embeddings.values()), ignore_index=True)
     st.markdown('<h6 class="sub-title">Select Dimensionality Reduction Method</h6>', unsafe_allow_html=True)
     reduction_method = st.selectbox("", options=["t-SNE", "PCA"], key=model_name)
     if reduction_method == "PCA":
@@ -176,125 +230,72 @@ def run_model(model_name):
     else:
         reducer = TSNE(n_components=2, random_state=42, perplexity=30, learning_rate=200)
     reduced = reducer.fit_transform(df_combined[embedding_cols].values)
-    
     dfs_reduced, unique_subsets = split_versions(df_combined, reduced)
-    selected_subsets = {"real": unique_subsets["real"], "es-digital-seq": unique_subsets["es-digital-seq"]}
-    color_maps = get_color_maps(selected_subsets)
     
-    fig, real_renderers, synthetic_renderers = create_figure(dfs_reduced, selected_subsets, color_maps)
-    centers_real = calculate_cluster_centers(dfs_reduced["real"], selected_subsets["real"])
-    centers_es = calculate_cluster_centers(dfs_reduced["es-digital-seq"], selected_subsets["es-digital-seq"])
-    df_distances = compute_distances(centers_es, centers_real)
+    # Se espera que unique_subsets tenga claves "real" y "synthetic"
+    color_maps = get_color_maps(unique_subsets)
+    fig, real_renderers, synthetic_renderers = create_figure(dfs_reduced, unique_subsets, color_maps)
+    
+    centers_real = calculate_cluster_centers(dfs_reduced["real"], unique_subsets["real"])
+    
+    df_distances = compute_wasserstein_distances_all_synthetics(dfs_reduced["synthetic"], 
+                                                                dfs_reduced["real"], 
+                                                                unique_subsets["real"])
     data_table, df_table, source_table = create_table(df_distances)
+    
     real_subset_names = list(df_table.columns[1:])
     real_select = Select(title="", value=real_subset_names[0], options=real_subset_names)
     reset_button = Button(label="Reset Colors", button_type="primary")
     line_source = ColumnDataSource(data={'x': [], 'y': []})
     fig.line('x', 'y', source=line_source, line_width=2, line_color='black')
     
-    synthetic_centers_js = {k: [v[0], v[1]] for k, v in centers_es.items()}
+    # Preparar centros para callback (para trazar líneas entre centros)
     real_centers_js = {k: [v[0], v[1]] for k, v in centers_real.items()}
     
-    # Callback para actualizar el gráfico
+    # Se podría preparar también los centros sintéticos si se requiere
+    synthetic_centers = {}
+    synth_labels = sorted(dfs_reduced["synthetic"]['label'].unique().tolist())
+    for label in synth_labels:
+        subset = dfs_reduced["synthetic"][dfs_reduced["synthetic"]['label'] == label]
+        synthetic_centers[label] = [subset['x'].mean(), subset['y'].mean()]
+    
     callback = CustomJS(args=dict(source=source_table, line_source=line_source,
-                                  synthetic_centers=synthetic_centers_js,
+                                  synthetic_centers=synthetic_centers,
                                   real_centers=real_centers_js,
-                                  synthetic_renderers=synthetic_renderers,
-                                  real_renderers=real_renderers,
-                                  synthetic_colors=color_maps["es-digital-seq"],
-                                  real_colors=color_maps["real"],
                                   real_select=real_select),
     code="""
         var selected = source.selected.indices;
         if (selected.length > 0) {
-            var row = selected[0];
+            var idx = selected[0];
             var data = source.data;
-            var synthetic_label = data['Synthetic'][row];
+            var synth_label = data['Synthetic'][idx];
             var real_label = real_select.value;
-            var syn_coords = synthetic_centers[synthetic_label];
+            var syn_coords = synthetic_centers[synth_label];
             var real_coords = real_centers[real_label];
-            line_source.data = { 'x': [syn_coords[0], real_coords[0]], 'y': [syn_coords[1], real_coords[1]] };
+            line_source.data = {'x': [syn_coords[0], real_coords[0]], 'y': [syn_coords[1], real_coords[1]]};
             line_source.change.emit();
-            
-            for (var key in synthetic_renderers) {
-                if (synthetic_renderers.hasOwnProperty(key)) {
-                    var renderer = synthetic_renderers[key];
-                    if (key === synthetic_label) {
-                        renderer.glyph.fill_color = synthetic_colors[key];
-                        renderer.glyph.line_color = synthetic_colors[key];
-                    } else {
-                        renderer.glyph.fill_color = "lightgray";
-                        renderer.glyph.line_color = "lightgray";
-                    }
-                }
-            }
-            for (var key in real_renderers) {
-                if (real_renderers.hasOwnProperty(key)) {
-                    var renderer = real_renderers[key];
-                    if (key === real_label) {
-                        renderer.glyph.fill_color = real_colors[key];
-                        renderer.glyph.line_color = real_colors[key];
-                    } else {
-                        renderer.glyph.fill_color = "lightgray";
-                        renderer.glyph.line_color = "lightgray";
-                    }
-                }
-            }
         } else {
-            line_source.data = { 'x': [], 'y': [] };
+            line_source.data = {'x': [], 'y': []};
             line_source.change.emit();
-            for (var key in synthetic_renderers) {
-                if (synthetic_renderers.hasOwnProperty(key)) {
-                    var renderer = synthetic_renderers[key];
-                    renderer.glyph.fill_color = synthetic_colors[key];
-                    renderer.glyph.line_color = synthetic_colors[key];
-                }
-            }
-            for (var key in real_renderers) {
-                if (real_renderers.hasOwnProperty(key)) {
-                    var renderer = real_renderers[key];
-                    renderer.glyph.fill_color = real_colors[key];
-                    renderer.glyph.line_color = real_colors[key];
-                }
-            }
         }
     """)
     source_table.selected.js_on_change('indices', callback)
     real_select.js_on_change('value', callback)
     
-    reset_callback = CustomJS(args=dict(line_source=line_source,
-                                        synthetic_renderers=synthetic_renderers,
-                                        real_renderers=real_renderers,
-                                        synthetic_colors=color_maps["es-digital-seq"],
-                                        real_colors=color_maps["real"]),
+    reset_callback = CustomJS(args=dict(line_source=line_source),
     code="""
-        line_source.data = { 'x': [], 'y': [] };
+        line_source.data = {'x': [], 'y': []};
         line_source.change.emit();
-        for (var key in synthetic_renderers) {
-            if (synthetic_renderers.hasOwnProperty(key)) {
-                var renderer = synthetic_renderers[key];
-                renderer.glyph.fill_color = synthetic_colors[key];
-                renderer.glyph.line_color = synthetic_colors[key];
-            }
-        }
-        for (var key in real_renderers) {
-            if (real_renderers.hasOwnProperty(key)) {
-                var renderer = real_renderers[key];
-                renderer.glyph.fill_color = real_colors[key];
-                renderer.glyph.line_color = real_colors[key];
-            }
-        }
     """)
     reset_button.js_on_event("button_click", reset_callback)
-
+    
     buffer = io.BytesIO()
     df_table.to_excel(buffer, index=False)
     buffer.seek(0)
-
+    
     layout = column(fig, column(real_select, reset_button, data_table))
     st.bokeh_chart(layout, use_container_width=True)
-
-    # Agregar un botón de descarga en Streamlit
+    
     st.download_button(
         label="Export Table",
         data=buffer,
@@ -302,18 +303,13 @@ def run_model(model_name):
         mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
         key=f"download_button_excel_{model_name}"
     )
-    
-
 
-# Función principal con tabs para cambiar de modelo
 def main():
     config_style()
     tabs = st.tabs(["Donut", "Idefics2"])
-    
     with tabs[0]:
         st.markdown('<h2 class="sub-title">Donut 🤗</h2>', unsafe_allow_html=True)
         run_model("Donut")
-    
     with tabs[1]:
         st.markdown('<h2 class="sub-title">Idefics2 🤗</h2>', unsafe_allow_html=True)
         run_model("Idefics2")