app.py

import gradio as gr
from datasets import load_dataset
import numpy as np
import model2vec
from reach import Reach
from difflib import ndiff

# Load the model at startup
model = model2vec.StaticModel.from_pretrained("minishlab/M2V_base_output")

# Default dataset parameters
default_dataset1_name = "sst2"
default_dataset1_split = "train"
default_dataset2_name = "sst2"
default_dataset2_split = "validation"
default_text_column = "sentence"
default_threshold = 0.9

# Load the default datasets at startup
ds_default1 = load_dataset(default_dataset1_name, split=default_dataset1_split)
ds_default2 = load_dataset(default_dataset2_name, split=default_dataset2_split)

def batch_iterable(iterable, batch_size):
    """Helper function to create batches from an iterable."""
    for i in range(0, len(iterable), batch_size):
        yield iterable[i:i + batch_size]

def compute_embeddings(texts, batch_size, progress, desc="Computing embeddings"):
    embeddings = []
    total_batches = (len(texts) + batch_size - 1) // batch_size
    for i, batch_texts in enumerate(batch_iterable(texts, batch_size)):
        batch_embeddings = model.encode(batch_texts, show_progressbar=False)
        embeddings.append(batch_embeddings)
        progress((i + 1) / total_batches, desc=desc)
    return np.concatenate(embeddings, axis=0)

def deduplicate(
    embedding_matrix: np.ndarray,
    threshold: float,
    batch_size: int = 1024,
    progress=None
) -> tuple[np.ndarray, dict[int, int]]:
    reach = Reach(vectors=embedding_matrix, items=[str(i) for i in range(len(embedding_matrix))])

    deduplicated_indices = set(range(len(embedding_matrix)))
    duplicate_to_original_mapping = {}

    results = reach.nearest_neighbor_threshold(
        embedding_matrix,
        threshold=threshold,
        batch_size=batch_size,
        show_progressbar=False,
    )

    total_items = len(embedding_matrix)
    for i, similar_items in enumerate(progress.tqdm(results, desc="Processing duplicates", total=total_items)):
        if i not in deduplicated_indices:
            continue

        similar_indices = [int(item[0]) for item in similar_items if int(item[0]) != i]
        for sim_idx in similar_indices:
            if sim_idx in deduplicated_indices:
                deduplicated_indices.remove(sim_idx)
                duplicate_to_original_mapping[sim_idx] = i

    return np.array(list(deduplicated_indices)), duplicate_to_original_mapping

def display_word_differences(x: str, y: str) -> str:
    diff = ndiff(x.split(), y.split())
    return " ".join([word for word in diff if word.startswith(("+", "-"))])

def perform_deduplication(
    deduplication_type,
    dataset1_name,
    dataset1_split,
    dataset1_text_column,
    dataset2_name="",
    dataset2_split="",
    dataset2_text_column="",
    threshold=default_threshold,
    progress=gr.Progress(track_tqdm=True),
):
    try:
        threshold = float(threshold)

        if deduplication_type == "Single dataset":
            ds = ds_default1 if dataset1_name == default_dataset1_name and dataset1_split == default_dataset1_split else load_dataset(dataset1_name, split=dataset1_split)
            texts = [example[dataset1_text_column] for example in ds]

            embedding_matrix = compute_embeddings(texts, batch_size=64, progress=progress)
            deduplicated_indices, duplicate_to_original_mapping = deduplicate(embedding_matrix, threshold, progress=progress)

            num_duplicates = len(duplicate_to_original_mapping)
            num_total = len(texts)
            num_deduplicated = len(deduplicated_indices)

            result_text = f"**Total documents:** {num_total}\n"
            result_text += f"**Number of duplicates found:** {num_duplicates}\n"
            result_text += f"**Number of unique documents after deduplication:** {num_deduplicated}\n\n"

            if num_duplicates > 0:
                result_text += "**Examples of duplicates found:**\n\n"
                num_examples = min(5, num_duplicates)
                for duplicate_idx, original_idx in list(duplicate_to_original_mapping.items())[:num_examples]:
                    original_text = texts[original_idx]
                    duplicate_text = texts[duplicate_idx]
                    differences = display_word_differences(original_text, duplicate_text)
                    result_text += f"**Original text:**\n{original_text}\n\n"
                    result_text += f"**Duplicate text:**\n{duplicate_text}\n\n"
                    result_text += f"**Differences:**\n{differences}\n"
                    result_text += "-" * 50 + "\n\n"
            else:
                result_text += "No duplicates found."

            yield result_text

    except Exception as e:
        yield f"An error occurred: {e}"

# Gradio interface setup
with gr.Blocks(css="#status_output { height: 150px; overflow: auto; }") as demo:
    gr.Markdown("# Semantic Deduplication")

    deduplication_type = gr.Radio(
        choices=["Single dataset", "Cross-dataset"],
        label="Deduplication Type",
        value="Single dataset",
    )

    with gr.Row():
        dataset1_name = gr.Textbox(value=default_dataset1_name, label="Dataset 1 Name")
        dataset1_split = gr.Textbox(value=default_dataset1_split, label="Dataset 1 Split")
        dataset1_text_column = gr.Textbox(value=default_text_column, label="Text Column Name")

    dataset2_inputs = gr.Column(visible=False)
    with dataset2_inputs:
        gr.Markdown("### Dataset 2")
        with gr.Row():
            dataset2_name = gr.Textbox(value=default_dataset2_name, label="Dataset 2 Name")
            dataset2_split = gr.Textbox(value=default_dataset2_split, label="Dataset 2 Split")
            dataset2_text_column = gr.Textbox(value=default_text_column, label="Text Column Name")

    threshold = gr.Slider(minimum=0.0, maximum=1.0, value=default_threshold, label="Similarity Threshold")

    compute_button = gr.Button("Compute")

    result_output = gr.Markdown()

    def update_visibility(deduplication_type_value):
        return gr.update(visible=True) if deduplication_type_value == "Cross-dataset" else gr.update(visible=False)

    deduplication_type.change(
        update_visibility, inputs=deduplication_type, outputs=dataset2_inputs
    )

    compute_button.click(
        fn=perform_deduplication,
        inputs=[
            deduplication_type,
            dataset1_name,
            dataset1_split,
            dataset1_text_column,
            dataset2_name,
            dataset2_split,
            dataset2_text_column,
            threshold,
        ],
        outputs=[result_output],
    )

demo.launch()


# import gradio as gr
# from datasets import load_dataset
# import numpy as np
# import model2vec
# from reach import Reach
# from difflib import ndiff
# import time

# # Load the model at startup
# model = model2vec.StaticModel.from_pretrained("minishlab/M2V_base_output")

# # Default dataset parameters
# default_dataset1_name = "sst2"
# default_dataset1_split = "train"
# default_dataset2_name = "sst2"
# default_dataset2_split = "validation"
# default_text_column = "sentence"
# default_threshold = 0.9

# # Load the default datasets at startup
# ds_default1 = load_dataset(default_dataset1_name, split=default_dataset1_split)
# ds_default2 = load_dataset(default_dataset2_name, split=default_dataset2_split)

# def batch_iterable(iterable, batch_size):
#     """Helper function to create batches from an iterable."""
#     for i in range(0, len(iterable), batch_size):
#         yield iterable[i:i + batch_size]

# def log_time(message, start_time=None, logs=None):
#     """Helper function to log the start and end times."""
#     current_time = time.time()
#     if start_time is not None:
#         elapsed = current_time - start_time
#         log_message = f"{message} - Took {elapsed:.2f} seconds"
#     else:
#         log_message = f"{message} - Started"
    
#     if logs is not None:
#         logs.append(log_message)

# def compute_embeddings(texts, batch_size, progress, logs, desc="Computing embeddings"):
#     embeddings = []
#     total_batches = (len(texts) + batch_size - 1) // batch_size
#     for i, batch_texts in enumerate(batch_iterable(texts, batch_size)):
#         batch_embeddings = model.encode(batch_texts, show_progressbar=False)
#         embeddings.append(batch_embeddings)
#         progress((i + 1) / total_batches, desc=desc)
#     return np.concatenate(embeddings, axis=0)

# def deduplicate(
#     embedding_matrix: np.ndarray,
#     threshold: float,
#     batch_size: int = 1024,
#     progress=None,
#     logs=None
# ) -> tuple[np.ndarray, dict[int, int]]:
#     # Building the index
#     log_time("Building search index", logs=logs)
#     reach = Reach(
#         vectors=embedding_matrix, items=[str(i) for i in range(len(embedding_matrix))]
#     )

#     deduplicated_indices = set(range(len(embedding_matrix)))
#     duplicate_to_original_mapping = {}

#     # Finding nearest neighbors
#     log_time("Finding nearest neighbors", logs=logs)
#     results = reach.nearest_neighbor_threshold(
#         embedding_matrix,
#         threshold=threshold,
#         batch_size=batch_size,
#         show_progressbar=False,  # Disable internal progress bar
#     )

#     # Processing duplicates with a progress bar
#     total_items = len(embedding_matrix)
#     log_time("Processing duplicates", logs=logs)
#     for i, similar_items in enumerate(
#         progress.tqdm(results, desc="Processing duplicates", total=total_items)
#     ):
#         if i not in deduplicated_indices:
#             continue

#         similar_indices = [int(item[0]) for item in similar_items if int(item[0]) != i]

#         for sim_idx in similar_indices:
#             if sim_idx in deduplicated_indices:
#                 deduplicated_indices.remove(sim_idx)
#                 duplicate_to_original_mapping[sim_idx] = i

#     return np.array(list(deduplicated_indices)), duplicate_to_original_mapping

# def display_word_differences(x: str, y: str) -> str:
#     diff = ndiff(x.split(), y.split())
#     return " ".join([word for word in diff if word.startswith(("+", "-"))])

# def encode_texts(texts, progress=None, logs=None):
#     embedding_matrix = model.encode(texts, show_progressbar=False)
#     log_time("Encoding texts completed", logs=logs)
#     return embedding_matrix

# def perform_deduplication(
#     deduplication_type,
#     dataset1_name,
#     dataset1_split,
#     dataset1_text_column,
#     dataset2_name="",
#     dataset2_split="",
#     dataset2_text_column="",
#     threshold=default_threshold,
#     progress=gr.Progress(track_tqdm=True),
# ):
#     logs = []  # To store log messages
#     try:
#         # Convert threshold to float
#         threshold = float(threshold)

#         # Initialize status message
#         log_time("Deduplication started", logs=logs)

#         if deduplication_type == "Single dataset":
#             # Load Dataset 1
#             start_time = time.time()
#             log_time("Loading Dataset 1", logs=logs)
#             if (
#                 dataset1_name == default_dataset1_name
#                 and dataset1_split == default_dataset1_split
#             ):
#                 ds = ds_default1
#             else:
#                 ds = load_dataset(dataset1_name, split=dataset1_split)
#             log_time("Loading Dataset 1 completed", start_time=start_time, logs=logs)

#             # Extract texts
#             start_time = time.time()
#             log_time("Extracting texts from Dataset 1", logs=logs)
#             texts = [example[dataset1_text_column] for example in ds]
#             log_time("Extracting texts from Dataset 1 completed", start_time=start_time, logs=logs)

#             # Compute embeddings
#             start_time = time.time()
#             log_time("Computing embeddings for Dataset 1", logs=logs)
#             embedding_matrix = encode_texts(texts, progress=progress, logs=logs)
#             log_time("Computing embeddings for Dataset 1 completed", start_time=start_time, logs=logs)

#             # Deduplicate
#             start_time = time.time()
#             log_time("Deduplicating embeddings", logs=logs)
#             deduplicated_indices, duplicate_to_original_mapping = deduplicate(
#                 embedding_matrix, threshold, progress=progress, logs=logs
#             )
#             log_time("Deduplication completed", start_time=start_time, logs=logs)

#             # Prepare the results
#             num_duplicates = len(duplicate_to_original_mapping)
#             num_total = len(texts)
#             num_deduplicated = len(deduplicated_indices)

#             result_text = f"**Total documents:** {num_total}\n"
#             result_text += f"**Number of duplicates found:** {num_duplicates}\n"
#             result_text += (
#                 f"**Number of unique documents after deduplication:** {num_deduplicated}\n\n"
#             )

#             # Show deduplicated examples
#             if num_duplicates > 0:
#                 result_text += "**Examples of duplicates found:**\n\n"
#                 num_examples = min(5, num_duplicates)
#                 for duplicate_idx, original_idx in list(duplicate_to_original_mapping.items())[:num_examples]:
#                     original_text = texts[original_idx]
#                     duplicate_text = texts[duplicate_idx]
#                     differences = display_word_differences(original_text, duplicate_text)
#                     result_text += f"**Original text:**\n{original_text}\n\n"
#                     result_text += f"**Duplicate text:**\n{duplicate_text}\n\n"
#                     result_text += f"**Differences:**\n{differences}\n"
#                     result_text += "-" * 50 + "\n\n"
#             else:
#                 result_text += "No duplicates found."

#             log_time("Deduplication process finished", logs=logs)
#             full_log = "\n".join(logs)  # Combine all logs into one output
#             yield full_log, result_text

#     except Exception as e:
#         full_log = "\n".join(logs)  # Combine all logs into one output in case of an error
#         yield f"An error occurred: {e}", ""
#         raise e

# # Adjust the height of the status_output component using custom CSS
# with gr.Blocks(css="#status_output { height: 150px; overflow: auto; }") as demo:
#     gr.Markdown("# Semantic Deduplication")

#     deduplication_type = gr.Radio(
#         choices=["Single dataset", "Cross-dataset"],
#         label="Deduplication Type",
#         value="Single dataset",
#     )

#     with gr.Row():
#         dataset1_name = gr.Textbox(value=default_dataset1_name, label="Dataset 1 Name")
#         dataset1_split = gr.Textbox(value=default_dataset1_split, label="Dataset 1 Split")
#         dataset1_text_column = gr.Textbox(value=default_text_column, label="Text Column Name")

#     dataset2_inputs = gr.Column(visible=False)
#     with dataset2_inputs:
#         gr.Markdown("### Dataset 2")
#         with gr.Row():
#             dataset2_name = gr.Textbox(value=default_dataset2_name, label="Dataset 2 Name")
#             dataset2_split = gr.Textbox(value=default_dataset2_split, label="Dataset 2 Split")
#             dataset2_text_column = gr.Textbox(value=default_text_column, label="Text Column Name")

#     threshold = gr.Slider(
#         minimum=0.0, maximum=1.0, value=default_threshold, label="Similarity Threshold"
#     )

#     compute_button = gr.Button("Compute")

#     # Use 'gr.Markdown' with 'elem_id' and custom CSS to adjust height
#     status_output = gr.Markdown(elem_id="status_output")
#     result_output = gr.Markdown()

#     # Function to update the visibility of dataset2_inputs
#     def update_visibility(deduplication_type_value):
#         if deduplication_type_value == "Cross-dataset":
#             return gr.update(visible=True)
#         else:
#             return gr.update(visible=False)

#     deduplication_type.change(
#         update_visibility, inputs=deduplication_type, outputs=dataset2_inputs
#     )

#     compute_button.click(
#         fn=perform_deduplication,
#         inputs=[
#             deduplication_type,
#             dataset1_name,
#             dataset1_split,
#             dataset1_text_column,
#             dataset2_name,
#             dataset2_split,
#             dataset2_text_column,
#             threshold,
#         ],
#         outputs=[status_output, result_output],
#     )

# demo.launch()



# # import gradio as gr
# # from datasets import load_dataset
# # import numpy as np
# # #from model2vec import StaticModel
# # import model2vec
# # from reach import Reach
# # from difflib import ndiff


# # # Load the model at startup
# # model = model2vec.StaticModel.from_pretrained("minishlab/M2V_base_output")

# # # Default dataset parameters
# # default_dataset1_name = "sst2"
# # default_dataset1_split = "train"
# # default_dataset2_name = "sst2"
# # default_dataset2_split = "validation"
# # default_text_column = "sentence"
# # default_threshold = 0.9

# # # Load the default datasets at startup
# # ds_default1 = load_dataset(default_dataset1_name, split=default_dataset1_split)
# # ds_default2 = load_dataset(default_dataset2_name, split=default_dataset2_split)


# # def batch_iterable(iterable, batch_size):
# #     """Helper function to create batches from an iterable."""
# #     for i in range(0, len(iterable), batch_size):
# #         yield iterable[i:i + batch_size]

# # def compute_embeddings(texts, batch_size, progress, desc="Computing embeddings"):
# #     embeddings = []
# #     total_batches = (len(texts) + batch_size - 1) // batch_size
# #     for i, batch_texts in enumerate(batch_iterable(texts, batch_size)):
# #         batch_embeddings = model.encode(batch_texts, show_progressbar=False)
# #         embeddings.append(batch_embeddings)
# #         progress((i + 1) / total_batches, desc=desc)
# #     return np.concatenate(embeddings, axis=0)

# # def deduplicate(
# #     embedding_matrix: np.ndarray,
# #     threshold: float,
# #     batch_size: int = 1024,
# #     progress=None
# # ) -> tuple[np.ndarray, dict[int, int]]:
# #     # Building the index
# #     progress(0, desc="Building search index...")
# #     reach = Reach(
# #         vectors=embedding_matrix, items=[str(i) for i in range(len(embedding_matrix))]
# #     )

# #     deduplicated_indices = set(range(len(embedding_matrix)))
# #     duplicate_to_original_mapping = {}

# #     # Finding nearest neighbors
# #     progress(0, desc="Finding nearest neighbors...")
# #     results = reach.nearest_neighbor_threshold(
# #         embedding_matrix,
# #         threshold=threshold,
# #         batch_size=batch_size,
# #         show_progressbar=False,  # Disable internal progress bar
# #     )

# #     # Processing duplicates with a progress bar
# #     total_items = len(embedding_matrix)
# #     for i, similar_items in enumerate(
# #         progress.tqdm(results, desc="Processing duplicates", total=total_items)
# #     ):
# #         if i not in deduplicated_indices:
# #             continue

# #         similar_indices = [int(item[0]) for item in similar_items if int(item[0]) != i]

# #         for sim_idx in similar_indices:
# #             if sim_idx in deduplicated_indices:
# #                 deduplicated_indices.remove(sim_idx)
# #                 duplicate_to_original_mapping[sim_idx] = i

# #     return np.array(list(deduplicated_indices)), duplicate_to_original_mapping

# # def display_word_differences(x: str, y: str) -> str:
# #     diff = ndiff(x.split(), y.split())
# #     return " ".join([word for word in diff if word.startswith(("+", "-"))])


# # def encode_texts(texts, progress=None):
# #     embedding_matrix = model.encode(texts, show_progressbar=False)
# #     return embedding_matrix

# # def perform_deduplication(
# #     deduplication_type,
# #     dataset1_name,
# #     dataset1_split,
# #     dataset1_text_column,
# #     dataset2_name="",
# #     dataset2_split="",
# #     dataset2_text_column="",
# #     threshold=default_threshold,
# #     progress=gr.Progress(track_tqdm=True),
# # ):
# #     try:
# #         # Convert threshold to float
# #         threshold = float(threshold)

# #         # Initialize status message
# #         status = ""

# #         if deduplication_type == "Single dataset":
# #             # Load Dataset 1
# #             status = "Loading Dataset 1..."
# #             yield status, ""
# #             if (
# #                 dataset1_name == default_dataset1_name
# #                 and dataset1_split == default_dataset1_split
# #             ):
# #                 ds = ds_default1
# #             else:
# #                 ds = load_dataset(dataset1_name, split=dataset1_split)

# #             # Extract texts
# #             status = "Extracting texts from Dataset 1..."
# #             yield status, ""
# #             texts = [example[dataset1_text_column] for example in ds]
# #             # Compute embeddings
# #             status = "Computing embeddings for Dataset 1..."
# #             yield status, ""
# #             embedding_matrix = encode_texts(texts, progress=progress)
# #             #embedding_matrix = model.encode(texts, show_progressbar=True)
# #             # embedding_matrix = compute_embeddings(
# #             #     texts,
# #             #     batch_size=64,
# #             #     progress=progress,
# #             #     desc="Computing embeddings for Dataset 1",
# #             # )

# #             # Deduplicate
# #             status = "Deduplicating embeddings..."
# #             yield status, ""
# #             deduplicated_indices, duplicate_to_original_mapping = deduplicate(
# #                 embedding_matrix, threshold, progress=progress
# #             )

# #             # Prepare the results
# #             num_duplicates = len(duplicate_to_original_mapping)
# #             num_total = len(texts)
# #             num_deduplicated = len(deduplicated_indices)

# #             result_text = f"**Total documents:** {num_total}\n"
# #             result_text += f"**Number of duplicates found:** {num_duplicates}\n"
# #             result_text += (
# #                 f"**Number of unique documents after deduplication:** {num_deduplicated}\n\n"
# #             )

# #             # Show deduplicated examples
# #             if num_duplicates > 0:
# #                 result_text += "**Examples of duplicates found:**\n\n"
# #                 num_examples = min(5, num_duplicates)
# #                 for duplicate_idx, original_idx in list(duplicate_to_original_mapping.items())[:num_examples]:
# #                     original_text = texts[original_idx]
# #                     duplicate_text = texts[duplicate_idx]
# #                     differences = display_word_differences(original_text, duplicate_text)
# #                     result_text += f"**Original text:**\n{original_text}\n\n"
# #                     result_text += f"**Duplicate text:**\n{duplicate_text}\n\n"
# #                     result_text += f"**Differences:**\n{differences}\n"
# #                     result_text += "-" * 50 + "\n\n"
# #             else:
# #                 result_text += "No duplicates found."

# #             # Final status
# #             status = "Deduplication completed."
# #             yield status, result_text

# #         elif deduplication_type == "Cross-dataset":
# #             # Similar code for cross-dataset deduplication
# #             # Load Dataset 1
# #             status = "Loading Dataset 1..."
# #             yield status, ""
# #             if (
# #                 dataset1_name == default_dataset1_name
# #                 and dataset1_split == default_dataset1_split
# #             ):
# #                 ds1 = ds_default1
# #             else:
# #                 ds1 = load_dataset(dataset1_name, split=dataset1_split)

# #             # Load Dataset 2
# #             status = "Loading Dataset 2..."
# #             yield status, ""
# #             if (
# #                 dataset2_name == default_dataset2_name
# #                 and dataset2_split == default_dataset2_split
# #             ):
# #                 ds2 = ds_default2
# #             else:
# #                 ds2 = load_dataset(dataset2_name, split=dataset2_split)

# #             # Extract texts from Dataset 1
# #             status = "Extracting texts from Dataset 1..."
# #             yield status, ""
# #             texts1 = [example[dataset1_text_column] for example in ds1]

# #             # Extract texts from Dataset 2
# #             status = "Extracting texts from Dataset 2..."
# #             yield status, ""
# #             texts2 = [example[dataset2_text_column] for example in ds2]

# #             # Compute embeddings for Dataset 1
# #             status = "Computing embeddings for Dataset 1..."
# #             yield status, ""
# #             embedding_matrix1 = compute_embeddings(
# #                 texts1,
# #                 batch_size=64,
# #                 progress=progress,
# #                 desc="Computing embeddings for Dataset 1",
# #             )

# #             # Compute embeddings for Dataset 2
# #             status = "Computing embeddings for Dataset 2..."
# #             yield status, ""
# #             embedding_matrix2 = compute_embeddings(
# #                 texts2,
# #                 batch_size=64,
# #                 progress=progress,
# #                 desc="Computing embeddings for Dataset 2",
# #             )

# #             # Deduplicate across datasets
# #             status = "Deduplicating embeddings across datasets..."
# #             yield status, ""
# #             duplicate_indices_in_ds2, duplicate_to_original_mapping = deduplicate_across_datasets(
# #                 embedding_matrix1, embedding_matrix2, threshold, progress=progress
# #             )

# #             num_duplicates = len(duplicate_indices_in_ds2)
# #             num_total_ds2 = len(texts2)
# #             num_unique_ds2 = num_total_ds2 - num_duplicates

# #             result_text = f"**Total documents in {dataset2_name}/{dataset2_split}:** {num_total_ds2}\n"
# #             result_text += f"**Number of duplicates found in {dataset2_name}/{dataset2_split}:** {num_duplicates}\n"
# #             result_text += f"**Number of unique documents in {dataset2_name}/{dataset2_split} after deduplication:** {num_unique_ds2}\n\n"

# #             # Show deduplicated examples
# #             if num_duplicates > 0:
# #                 result_text += "**Examples of duplicates found in Dataset 2:**\n\n"
# #                 num_examples = min(5, num_duplicates)
# #                 for duplicate_idx in duplicate_indices_in_ds2[:num_examples]:
# #                     original_idx = duplicate_to_original_mapping[duplicate_idx]
# #                     original_text = texts1[original_idx]
# #                     duplicate_text = texts2[duplicate_idx]
# #                     differences = display_word_differences(original_text, duplicate_text)
# #                     result_text += f"**Original text (Dataset 1):**\n{original_text}\n\n"
# #                     result_text += f"**Duplicate text (Dataset 2):**\n{duplicate_text}\n\n"
# #                     result_text += f"**Differences:**\n{differences}\n"
# #                     result_text += "-" * 50 + "\n\n"
# #             else:
# #                 result_text += "No duplicates found."

# #             # Final status
# #             status = "Deduplication completed."
# #             yield status, result_text

# #     except Exception as e:
# #         yield f"An error occurred: {e}", ""
# #         raise e

# # def deduplicate_across_datasets(
# #     embedding_matrix_1: np.ndarray,
# #     embedding_matrix_2: np.ndarray,
# #     threshold: float,
# #     batch_size: int = 1024,
# #     progress=None
# # ) -> tuple[list[int], dict[int, int]]:
# #     # Building the index from Dataset 1
# #     progress(0, desc="Building search index from Dataset 1...")
# #     reach = Reach(
# #         vectors=embedding_matrix_1, items=[str(i) for i in range(len(embedding_matrix_1))]
# #     )

# #     duplicate_indices_in_test = []
# #     duplicate_to_original_mapping = {}

# #     # Finding nearest neighbors between datasets
# #     progress(0, desc="Finding nearest neighbors between datasets...")
# #     results = reach.nearest_neighbor_threshold(
# #         embedding_matrix_2,
# #         threshold=threshold,
# #         batch_size=batch_size,
# #         show_progressbar=False,  # Disable internal progress bar
# #     )

# #     total_items = len(embedding_matrix_2)
# #     # Processing duplicates with a progress bar
# #     for i, similar_items in enumerate(
# #         progress.tqdm(results, desc="Processing duplicates across datasets", total=total_items)
# #     ):
# #         similar_indices = [int(item[0]) for item in similar_items if item[1] >= threshold]

# #         if similar_indices:
# #             duplicate_indices_in_test.append(i)
# #             duplicate_to_original_mapping[i] = similar_indices[0]

# #     return duplicate_indices_in_test, duplicate_to_original_mapping

# # # Adjust the height of the status_output component using custom CSS
# # with gr.Blocks(css="#status_output { height: 150px; overflow: auto; }") as demo:
# #     gr.Markdown("# Semantic Deduplication")

# #     deduplication_type = gr.Radio(
# #         choices=["Single dataset", "Cross-dataset"],
# #         label="Deduplication Type",
# #         value="Single dataset",
# #     )

# #     with gr.Row():
# #         dataset1_name = gr.Textbox(value=default_dataset1_name, label="Dataset 1 Name")
# #         dataset1_split = gr.Textbox(value=default_dataset1_split, label="Dataset 1 Split")
# #         dataset1_text_column = gr.Textbox(value=default_text_column, label="Text Column Name")

# #     dataset2_inputs = gr.Column(visible=False)
# #     with dataset2_inputs:
# #         gr.Markdown("### Dataset 2")
# #         with gr.Row():
# #             dataset2_name = gr.Textbox(value=default_dataset2_name, label="Dataset 2 Name")
# #             dataset2_split = gr.Textbox(value=default_dataset2_split, label="Dataset 2 Split")
# #             dataset2_text_column = gr.Textbox(value=default_text_column, label="Text Column Name")

# #     threshold = gr.Slider(
# #         minimum=0.0, maximum=1.0, value=default_threshold, label="Similarity Threshold"
# #     )

# #     compute_button = gr.Button("Compute")

# #     # Use 'gr.Markdown' with 'elem_id' and custom CSS to adjust height
# #     status_output = gr.Markdown(elem_id="status_output")
# #     result_output = gr.Markdown()

# #     # Function to update the visibility of dataset2_inputs
# #     def update_visibility(deduplication_type_value):
# #         if deduplication_type_value == "Cross-dataset":
# #             return gr.update(visible=True)
# #         else:
# #             return gr.update(visible=False)

# #     deduplication_type.change(
# #         update_visibility, inputs=deduplication_type, outputs=dataset2_inputs
# #     )

# #     compute_button.click(
# #         fn=perform_deduplication,
# #         inputs=[
# #             deduplication_type,
# #             dataset1_name,
# #             dataset1_split,
# #             dataset1_text_column,
# #             dataset2_name,
# #             dataset2_split,
# #             dataset2_text_column,
# #             threshold,
# #         ],
# #         outputs=[status_output, result_output],
# #     )

# # demo.launch()