File size: 33,875 Bytes
4bf5701
 
 
 
8d064dc
0bbf2df
b4c92f5
b44d470
3011301
b4c92f5
 
 
4bf5701
 
bda2b5b
 
0bbf2df
4bf5701
 
 
 
28a74a6
4bf5701
 
 
b44d470
4bf5701
b44d470
4bf5701
 
 
 
 
28a74a6
4bf5701
 
 
0bbf2df
4bf5701
 
 
 
 
 
 
 
b4c92f5
 
 
 
4bf5701
28a74a6
4bf5701
0bbf2df
 
4bf5701
 
 
 
b4c92f5
4bf5701
ce093f9
0bbf2df
4bf5701
 
b4c92f5
 
b2ce320
 
 
 
3011301
b2ce320
bda2b5b
b4c92f5
 
b44d470
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b4c92f5
 
 
 
 
 
 
 
 
 
 
4bf5701
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b4c92f5
 
 
 
edd4b9d
ce093f9
b4c92f5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4bf5701
 
b4c92f5
 
 
988e855
4bf5701
 
 
 
 
b4c92f5
 
 
0bbf2df
b4c92f5
 
 
4bf5701
 
 
 
 
 
 
 
 
0bbf2df
 
 
4bf5701
0bbf2df
b4c92f5
a34433b
b4c92f5
a34433b
 
 
 
b4c92f5
a34433b
 
 
 
b4c92f5
a34433b
4bf5701
a34433b
4bf5701
 
b4c92f5
 
 
 
 
 
 
4bf5701
 
 
 
b4c92f5
4bf5701
bda2b5b
 
988e855
 
 
 
 
 
 
 
 
 
 
 
bda2b5b
 
988e855
bda2b5b
 
 
988e855
 
 
 
4bf5701
b4c92f5
 
 
 
 
 
 
 
 
 
 
 
 
4bf5701
b4c92f5
 
 
 
 
 
 
 
 
 
 
ce093f9
b4c92f5
 
 
 
 
28a74a6
b4c92f5
bda2b5b
4bf5701
 
 
 
 
 
 
 
 
 
59185b1
b4c92f5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0bbf2df
 
 
 
 
 
 
 
bda2b5b
 
 
0bbf2df
4bf5701
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b4c92f5
4bf5701
 
 
0bbf2df
 
 
 
 
 
 
4bf5701
b4c92f5
 
 
 
 
 
 
 
 
 
 
4bf5701
 
b4c92f5
 
 
 
 
 
 
4bf5701
 
 
b4c92f5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3011301
 
 
 
 
 
a7f196f
 
3011301
4bf5701
59185b1
 
b44d470
 
b4c92f5
 
b44d470
 
59185b1
4bf5701
 
 
 
 
 
 
 
 
0bbf2df
4bf5701
b4c92f5
 
 
 
 
 
 
4bf5701
 
 
 
b4c92f5
 
 
 
 
 
 
4bf5701
b4c92f5
bda2b5b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b4c92f5
 
 
 
 
 
4bf5701
 
b4c92f5
 
4bf5701
 
 
 
 
 
b4c92f5
4bf5701
edd4b9d
4bf5701
 
 
 
 
0bbf2df
 
 
 
4bf5701
 
 
 
 
 
 
 
 
bda2b5b
 
 
 
4bf5701
 
b4c92f5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8d064dc
 
b4c92f5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4bf5701
b4c92f5
59185b1
4bf5701
 
 
 
 
 
b4c92f5
4bf5701
bda2b5b
 
 
59185b1
b4c92f5
 
 
 
 
 
 
4bf5701
b4c92f5
4bf5701
 
 
 
 
59185b1
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
import gradio as gr
from pathlib import Path
from pipeline.process import process_texts
from pipeline.visualize import generate_visualizations, generate_word_count_chart
from pipeline.llm_service import LLMService
import logging
import pandas as pd
from datetime import datetime

from dotenv import load_dotenv

# Load environment variables from .env file
from theme import tibetan_theme

load_dotenv()

logger = logging.getLogger(__name__)
def main_interface():
    with gr.Blocks(
        theme=tibetan_theme,
        title="Tibetan Text Metrics Web App",
        css=tibetan_theme.get_css_string() + ".metric-description, .step-box { padding: 1.5rem !important; }"
    ) as demo:
        gr.Markdown(
            """# Tibetan Text Metrics Web App
<span style='font-size:18px;'>A user-friendly web application for analyzing textual similarities and variations in Tibetan manuscripts, providing a graphical interface to the core functionalities of the [Tibetan Text Metrics (TTM)](https://github.com/daniel-wojahn/tibetan-text-metrics) project. Powered by advanced language models via OpenRouter for in-depth text analysis.</span>
        """,

            elem_classes="gr-markdown",
        )

        with gr.Row(elem_id="steps-row"):
            with gr.Column(scale=1, elem_classes="step-column"):
                with gr.Group(elem_classes="step-box"):
                    gr.Markdown(
                        """
                    ## Step 1: Upload Your Tibetan Text Files
                    <span style='font-size:16px;'>Upload one or more `.txt` files. Each file should contain Unicode Tibetan text, segmented into chapters/sections if possible using the marker '༈' (<i>sbrul shad</i>).</span>
                    """,
                        elem_classes="gr-markdown",
                    )
                    file_input = gr.File(
                        label="Upload Tibetan .txt files",
                        file_types=[".txt"],
                        file_count="multiple",
                    )
                    gr.Markdown(
                        "<small>Note: Maximum file size: 10MB per file. For optimal performance, use files under 1MB.</small>",
                        elem_classes="gr-markdown"
                    )
            with gr.Column(scale=1, elem_classes="step-column"):
                with gr.Group(elem_classes="step-box"):
                    gr.Markdown(
                        """## Step 2: Configure and run the analysis
<span style='font-size:16px;'>Choose your analysis options and click the button below to compute metrics and view results. For meaningful analysis, ensure your texts are segmented by chapter or section using the marker '༈' (<i>sbrul shad</i>). The tool will split files based on this marker.</span>
                    """,
                        elem_classes="gr-markdown",
                    )
                    semantic_toggle_radio = gr.Radio(
                        label="Compute semantic similarity? (Experimental)",
                        choices=["Yes", "No"],
                        value="No",
                        info="Semantic similarity will be time-consuming. Choose 'No' to speed up analysis if these metrics are not required.",
                        elem_id="semantic-radio-group",
                    )
                    
                    model_dropdown = gr.Dropdown(
                        choices=[
                            "sentence-transformers/LaBSE", 
                            "Facebook FastText (Pre-trained)"
                        ],
                        label="Select Embedding Model",
                        value="sentence-transformers/LaBSE",
                        info="Select the embedding model to use for semantic similarity analysis. LaBSE v1.0 or FastText v1.0."
                    )
                    
                    with gr.Accordion("Advanced Options", open=False):
                        batch_size_slider = gr.Slider(
                            minimum=1,
                            maximum=64,
                            value=8,
                            step=1,
                            label="Batch Size (for Hugging Face models)",
                            info="Adjust based on your hardware (VRAM). Lower this if you encounter memory issues."
                        )
                        progress_bar_checkbox = gr.Checkbox(
                            label="Show Embedding Progress Bar",
                            value=False,
                            info="Display a progress bar during embedding generation. Useful for large datasets."
                        )

                    stopwords_dropdown = gr.Dropdown(
                        label="Stopword Filtering",
                        choices=[
                            "None (No filtering)", 
                            "Standard (Common particles only)", 
                            "Aggressive (All function words)"
                        ],
                        value="Standard (Common particles only)",  # Default
                        info="Choose how aggressively to filter out common Tibetan particles and function words when calculating similarity. This helps focus on meaningful content words."
                    )

                    process_btn = gr.Button(
                        "Run Analysis", elem_id="run-btn", variant="primary"
                    )

        gr.Markdown(
            """## Results
        """,
            elem_classes="gr-markdown",
        )
        # The heatmap_titles and metric_tooltips dictionaries are defined here
        # heatmap_titles = { ... }
        # metric_tooltips = { ... }
        csv_output = gr.File(label="Download CSV Results")
        metrics_preview = gr.Dataframe(
            label="Similarity Metrics Preview", interactive=False, visible=True
        )
        
        # LLM Interpretation components
        with gr.Row():
            with gr.Column():
                gr.Markdown(
                    "## AI Analysis\n*The AI will analyze your text similarities and provide insights into patterns and relationships.*",
                    elem_classes="gr-markdown"
                )
                
                # Add the interpret button
                with gr.Row():
                    interpret_btn = gr.Button(
                        "Help Interpret Results",
                        variant="primary",
                        elem_id="interpret-btn"
                    )
                # Create a placeholder message with proper formatting and structure
                initial_message = """
## Analysis of Tibetan Text Similarity Metrics

<small>*Click the 'Help Interpret Results' button above to generate an AI-powered analysis of your similarity metrics.*</small>
"""
                interpretation_output = gr.Markdown(
                    value=initial_message,
                    elem_id="llm-analysis"
                )
        
        # Heatmap tabs for each metric
        heatmap_titles = {
            "Jaccard Similarity (%)": "Jaccard Similarity (%): Higher scores (darker) mean more shared unique words.",
            "Normalized LCS": "Normalized LCS: Higher scores (darker) mean longer shared sequences of words.",
            "Semantic Similarity": "Semantic Similarity (using word embeddings/experimental): Higher scores (darker) mean more similar meanings.",
            "Word Counts": "Word Counts: Bar chart showing the number of words in each segment after tokenization.",
        }

        metric_tooltips = {
            "Jaccard Similarity (%)": """
### Jaccard Similarity (%)
This metric quantifies the lexical overlap between two text segments by comparing their sets of *unique* words, optionally filtering out common Tibetan stopwords. 

It essentially answers the question: 'Of all the distinct words found across these two segments, what proportion of them are present in both?' It is calculated as `(Number of common unique words) / (Total number of unique words in both texts combined) * 100`. 

Jaccard Similarity is insensitive to word order and word frequency; it only cares whether a unique word is present or absent. A higher percentage indicates a greater overlap in the vocabularies used in the two segments.

**Stopword Filtering**: When enabled (via the "Filter Stopwords" checkbox), common Tibetan particles and function words are filtered out before comparison. This helps focus on meaningful content words rather than grammatical elements.
""",
            "Normalized LCS": """
### Normalized LCS (Longest Common Subsequence)
This metric measures the length of the longest sequence of words that appears in *both* text segments, maintaining their original relative order. 
Importantly, these words do not need to be directly adjacent (contiguous) in either text. 
For example, if Text A is '<u>the</u> quick <u>brown</u> fox <u>jumps</u>' and Text B is '<u>the</u> lazy cat and <u>brown</u> dog <u>jumps</u> high', the LCS is 'the brown jumps'. 
The length of this common subsequence is then normalized (in this tool, by dividing by the length of the longer of the two segments) to provide a score, which is then presented as a percentage. 
A higher Normalized LCS score suggests more significant shared phrasing, direct textual borrowing, or strong structural parallelism, as it reflects similarities in how ideas are ordered and expressed sequentially.

**No Stopword Filtering.** Unlike metrics such as Jaccard Similarity or TF-IDF Cosine Similarity (which typically filter out common stopwords to focus on content-bearing words), the LCS calculation in this tool intentionally uses the raw, unfiltered sequence of tokens from your texts. This design choice allows LCS to capture structural similarities and the flow of language, including the use of particles and common words that contribute to sentence construction and narrative sequence. By not removing stopwords, LCS can reveal similarities in phrasing and textual structure that might otherwise be obscured, making it a valuable complement to metrics that focus purely on lexical overlap of keywords.

**Note on Interpretation**: It is possible for Normalized LCS to be higher than Jaccard Similarity. This often happens when texts share a substantial 'narrative backbone' or common ordered phrases (leading to a high LCS), even if they use varied surrounding vocabulary or introduce many unique words not part of these core sequences (which would lower the Jaccard score). LCS highlights this sequential, structural similarity, while Jaccard focuses on the overall shared vocabulary regardless of its arrangement.
""",
            "Semantic Similarity": """
### Semantic Similarity
This metric measures the similarity in meaning between text segments by converting them into high-dimensional vectors (embeddings) and calculating the cosine similarity between them. A score closer to 1 indicates a higher degree of semantic overlap. You can choose from two types of models:

**1. Hugging Face Transformer Models (e.g., LaBSE, E5-base-v2)**:
   - **Context-Aware**: These are state-of-the-art models that understand the context in which words appear, allowing them to capture nuanced meanings, relationships, and idiomatic expressions.
   - **Sentence-Level Embeddings**: They are specifically designed to create a single, high-quality vector representation for an entire text segment.
   - **Recommended for General Use**: These models often provide superior performance for capturing the overall thematic and semantic similarity between passages.

**2. FastText Model**:
   - **Word-Based**: This model generates embeddings for individual words and then averages them (using TF-IDF weighting) to create a representation for the entire segment.
   - **Strong for Lexical Analysis**: It excels at understanding the meaning of individual words, including rare or specialized Tibetan vocabulary, as it can generate vectors for out-of-vocabulary words from their sub-word units.
   - **Optimized for Tibetan**: Uses `botok` tokenization and TF-IDF weighting to focus on the most significant words in a segment.

**Stopword Filtering**: When enabled, common Tibetan particles and function words are filtered out before computing embeddings. This helps focus on meaningful content words and is recommended for most analyses.
""",

            "TF-IDF Cosine Sim": """
### TF-IDF Cosine Similarity
This metric calculates Term Frequency-Inverse Document Frequency (TF-IDF) scores for each word in each text segment, optionally filtering out common Tibetan stopwords. 

TF-IDF gives higher weight to words that are frequent within a particular segment but relatively rare across the entire collection of segments. This helps identify terms that are characteristic or discriminative for a segment. When stopword filtering is enabled, the TF-IDF scores better reflect genuinely significant terms by excluding common particles and function words.

Each segment is represented as a vector of these TF-IDF scores, and the cosine similarity is computed between these vectors. A score closer to 1 indicates that the two segments share more important, distinguishing terms, suggesting they cover similar specific topics or themes.

**Stopword Filtering**: When enabled (via the "Filter Stopwords" checkbox), common Tibetan particles and function words are filtered out. This can be toggled on/off to compare results with and without stopwords.
""",
        }
        heatmap_tabs = {}
        gr.Markdown("## Detailed Metric Analysis", elem_classes="gr-markdown")
        
        with gr.Tabs(elem_id="heatmap-tab-group"):
            # Structural Analysis Tab
            with gr.Tab("Structural Analysis"):
                gr.Markdown("""
                ### Structural Analysis for Tibetan Legal Manuscripts
                
                This tab provides detailed chapter-level structural analysis for Tibetan legal manuscript comparison.
                
                **Features:**
                - **Differential Highlighting**: Highlights significant textual variations
                - **Per-Chapter Analysis**: Detailed comparison for each chapter pair
                
                **Usage:**
                Results appear automatically when texts are processed. Use the export buttons to save detailed reports for philological analysis.
                """)
                
                # Structural analysis outputs
                structural_heatmap = gr.Plot(label="Structural Changes Summary", show_label=False, elem_classes="structural-heatmap")
                structural_report = gr.HTML(label="Differential Analysis Report")
                structural_export = gr.File(label="Export Structural Analysis Report", file_types=[".html", ".md", ".json"])
                
            # Process metrics excluding TF-IDF
            metrics_to_display = {k: v for k, v in heatmap_titles.items() if k != "TF-IDF Cosine Sim"}
            
            for metric_key, descriptive_title in metrics_to_display.items():
                with gr.Tab(metric_key):
                    # Set CSS class based on metric type
                    if metric_key == "Jaccard Similarity (%)":
                        css_class = "metric-info-accordion jaccard-info"
                        accordion_title = "Understanding Vocabulary Overlap"
                    elif metric_key == "Normalized LCS":
                        css_class = "metric-info-accordion lcs-info"
                        accordion_title = "Understanding Sequence Patterns"
                    elif metric_key == "Semantic Similarity":
                        css_class = "metric-info-accordion semantic-info"
                        accordion_title = "Understanding Meaning Similarity"
                    elif metric_key == "Word Counts":
                        css_class = "metric-info-accordion wordcount-info"
                        accordion_title = "Understanding Text Length"
                    else:
                        css_class = "metric-info-accordion"
                        accordion_title = f"About {metric_key}"
                    
                    # Create the accordion with appropriate content
                    with gr.Accordion(accordion_title, open=False, elem_classes=css_class):
                        if metric_key == "Word Counts":
                            gr.Markdown("""
                            ### Word Counts per Segment
                            This chart displays the number of words in each segment of your texts after tokenization.
                            """)
                        elif metric_key in metric_tooltips:
                            gr.Markdown(value=metric_tooltips[metric_key], elem_classes="metric-description")
                        else:
                            gr.Markdown(value=f"### {metric_key}\nDescription not found.")
                    
                    # Add the appropriate plot
                    if metric_key == "Word Counts":
                        word_count_plot = gr.Plot(label="Word Counts per Segment", show_label=False, scale=1, elem_classes="metric-description")
                    else:
                        heatmap_tabs[metric_key] = gr.Plot(label=f"Heatmap: {metric_key}", show_label=False, elem_classes="metric-heatmap")

        # The outputs in process_btn.click should use the short metric names as keys for heatmap_tabs
        # e.g., heatmap_tabs["Jaccard Similarity (%)"]
        # Ensure the plot is part of the layout. This assumes plots are displayed sequentially
        # within the current gr.Tab("Results"). If they are in specific TabItems, this needs adjustment.
        # For now, this modification focuses on creating the plot object and making it an output.
        # The visual placement depends on how Gradio renders children of gr.Tab or if there's another container.

        warning_box = gr.Markdown(visible=False)

        def run_pipeline(files, enable_semantic, model_name, stopwords_option, batch_size, show_progress, progress=gr.Progress()):
            """Run the text analysis pipeline on the uploaded files.

            Args:
                files: List of uploaded files
                enable_semantic: Whether to compute semantic similarity
                model_name: Name of the embedding model to use
                stopwords_option: Stopword filtering level (None, Standard, or Aggressive)
                progress: Gradio progress indicator

            Returns:
                Tuple of (metrics_df, heatmap_jaccard, heatmap_lcs, heatmap_semantic, heatmap_tfidf, word_count_fig)
            """
            # Initialize progress tracking
            try:
                progress_tracker = gr.Progress()
            except Exception as e:
                logger.warning(f"Could not initialize progress tracker: {e}")
                progress_tracker = None
            # Initialize all return values to ensure defined paths for all outputs
            csv_path_res = None
            metrics_preview_df_res = None # Can be a DataFrame or a string message
            word_count_fig_res = None
            jaccard_heatmap_res = None
            lcs_heatmap_res = None
            semantic_heatmap_res = None
            warning_update_res = gr.update(value="", visible=False) # Default: no warning
            structural_heatmap_res = None
            structural_report_res = None
            structural_export_res = None

            """
            Processes uploaded files, computes metrics, generates visualizations, and prepares outputs for the UI.

            Args:
                files (List[FileStorage]): A list of file objects uploaded by the user.

            Returns:
                tuple: A tuple containing the following elements in order:
                    - csv_path (str | None): Path to the generated CSV results file, or None on error.
                    - metrics_preview_df (pd.DataFrame | str | None): DataFrame for metrics preview, error string, or None.
                    - word_count_fig (matplotlib.figure.Figure | None): Plot of word counts, or None on error.
                    - jaccard_heatmap (matplotlib.figure.Figure | None): Jaccard similarity heatmap, or None.
                    - lcs_heatmap (matplotlib.figure.Figure | None): LCS heatmap, or None.
                    - semantic_heatmap (matplotlib.figure.Figure | None): Semantic similarity heatmap, or None.
                    - warning_update (gr.update): Gradio update for the warning box.
            """
            # Check if files are provided
            if not files:
                return (
                    None,
                    "Please upload files to analyze.",
                    None,  # word_count_plot
                    None,  # jaccard_heatmap
                    None,  # lcs_heatmap
                    None,  # semantic_heatmap
                    None,  # tfidf_heatmap
                    gr.update(value="Please upload files.", visible=True),
                )
                
            # Check file size limits (10MB per file)
            for file in files:
                file_size_mb = Path(file.name).stat().st_size / (1024 * 1024)
                if file_size_mb > 10:
                    return (
                        None,
                        f"File '{Path(file.name).name}' exceeds the 10MB size limit (size: {file_size_mb:.2f}MB).",
                        None, None, None, None, None,
                        gr.update(value=f"Error: File '{Path(file.name).name}' exceeds the 10MB size limit.", visible=True),
                    )

            try:
                if progress_tracker is not None:
                    try:
                        progress_tracker(0.1, desc="Preparing files...")
                    except Exception as e:
                        logger.warning(f"Progress update error (non-critical): {e}")
                
                # Get filenames and read file contents
                filenames = [
                    Path(file.name).name for file in files
                ]  # Use Path().name to get just the filename
                text_data = {}
                
                # Read files with progress updates
                for i, file in enumerate(files):
                    file_path = Path(file.name)
                    filename = file_path.name
                    if progress_tracker is not None:
                        try:
                            progress_tracker(0.1 + (0.1 * (i / len(files))), desc=f"Reading file: {filename}")
                        except Exception as e:
                            logger.warning(f"Progress update error (non-critical): {e}")
                    
                    try:
                        text_data[filename] = file_path.read_text(encoding="utf-8-sig")
                    except UnicodeDecodeError:
                        # Try with different encodings if UTF-8 fails
                        try:
                            text_data[filename] = file_path.read_text(encoding="utf-16")
                        except UnicodeDecodeError:
                            return (
                                None,
                                f"Error: Could not decode file '{filename}'. Please ensure it contains valid Tibetan text in UTF-8 or UTF-16 encoding.",
                                None, None, None, None, None,
                                gr.update(value=f"Error: Could not decode file '{filename}'.", visible=True),
                            )

                # Configure semantic similarity
                enable_semantic_bool = enable_semantic == "Yes"
                
                if progress_tracker is not None:
                    try:
                        progress_tracker(0.2, desc="Loading model..." if enable_semantic_bool else "Processing text...")
                    except Exception as e:
                        logger.warning(f"Progress update error (non-critical): {e}")
                
                # Process texts with selected model
                # Convert stopword option to appropriate parameters
                use_stopwords = stopwords_option != "None (No filtering)"
                use_lite_stopwords = stopwords_option == "Standard (Common particles only)"
                
                # Map UI model name to internal model ID
                # The UI model_name is "Facebook FastText (Pre-trained)"
                # This mapping ensures the backend receives the correct identifier.
                if model_name == "Facebook FastText (Pre-trained)":
                    internal_model_id = "facebook-fasttext-pretrained"
                else:
                    # For Hugging Face models, the UI value is the correct model ID
                    internal_model_id = model_name

                df_results, word_counts_df_data, warning_raw = process_texts(
                    text_data,
                    filenames,
                    enable_semantic=enable_semantic_bool,
                    model_name=internal_model_id,
                    use_stopwords=use_stopwords,
                    use_lite_stopwords=use_lite_stopwords,
                    progress_callback=progress_tracker,
                    batch_size=batch_size,
                    show_progress_bar=show_progress
                )

                if df_results.empty:
                    warning_md = f"**⚠️ Warning:** {warning_raw}" if warning_raw else ""
                    warning_message = (
                        "No common chapters found or results are empty. " + warning_md
                    )
                    metrics_preview_df_res = warning_message
                    warning_update_res = gr.update(value=warning_message, visible=True)
                    # Results for this case are set, then return
                else:
                    # Generate visualizations
                    if progress_tracker is not None:
                        try:
                            progress_tracker(0.8, desc="Generating visualizations...")
                        except Exception as e:
                            logger.warning(f"Progress update error (non-critical): {e}")
                    
                    # heatmap_titles is already defined in the outer scope of main_interface
                    heatmaps_data = generate_visualizations(
                        df_results, descriptive_titles=heatmap_titles
                    )
                    
                    # Generate word count chart
                    if progress_tracker is not None:
                        try:
                            progress_tracker(0.9, desc="Creating word count chart...")
                        except Exception as e:
                            logger.warning(f"Progress update error (non-critical): {e}")
                    word_count_fig_res = generate_word_count_chart(word_counts_df_data)
                    
                    # Generate structural analysis
                    if progress_tracker is not None:
                        try:
                            progress_tracker(0.92, desc="Generating structural analysis...")
                        except Exception as e:
                            logger.warning(f"Progress update error (non-critical): {e}")
                    
                    # Create structural analysis
                    from pipeline.differential_viz import create_differential_heatmap, create_change_detection_report
                    
                    # Create structural heatmap
                    try:
                        structural_heatmap_res = create_differential_heatmap(
                            text_data, "all_chapters", df_results
                        )
                    except Exception as e:
                        logger.warning(f"Could not generate structural heatmap: {e}")
                        structural_heatmap_res = None
                    
                    # Create structural report
                    try:
                        structural_report_res = create_change_detection_report(
                            text_data, "all_chapters", "html"
                        )
                    except Exception as e:
                        logger.warning(f"Could not generate structural report: {e}")
                        structural_report_res = "<p>Could not generate structural analysis report.</p>"
                    
                    # Save structural analysis report
                    try:
                        report_path = "structural_analysis_report.html"
                        with open(report_path, 'w', encoding='utf-8') as f:
                            f.write(structural_report_res if isinstance(structural_report_res, str) else "")
                        structural_export_res = report_path
                    except Exception as e:
                        logger.warning(f"Could not save structural report: {e}")
                        structural_export_res = None
                    
                    # Save results to CSV
                    if progress_tracker is not None:
                        try:
                            progress_tracker(0.95, desc="Saving results...")
                        except Exception as e:
                            logger.warning(f"Progress update error (non-critical): {e}")
                    csv_path_res = "results.csv"
                    df_results.to_csv(csv_path_res, index=False)
                    
                    # Prepare final output
                    warning_md = f"**⚠️ Warning:** {warning_raw}" if warning_raw else ""
                    metrics_preview_df_res = df_results.head(10)

                    jaccard_heatmap_res = heatmaps_data.get("Jaccard Similarity (%)")
                    lcs_heatmap_res = heatmaps_data.get("Normalized LCS")
                    semantic_heatmap_res = heatmaps_data.get(
                        "Semantic Similarity"
                    )
                    _ = heatmaps_data.get("TF-IDF Cosine Sim")  # TF-IDF removed
                    warning_update_res = gr.update(
                        visible=bool(warning_raw), value=warning_md
                    )

            except Exception as e:
                logger.error(f"Error in run_pipeline: {e}", exc_info=True)
                # metrics_preview_df_res and warning_update_res are set here.
                # Other plot/file path variables will retain their initial 'None' values set at function start.
                metrics_preview_df_res = f"Error: {str(e)}" 
                warning_update_res = gr.update(value=f"Error: {str(e)}", visible=True)

            return (
                csv_path_res,
                metrics_preview_df_res,
                word_count_fig_res,
                jaccard_heatmap_res,
                lcs_heatmap_res,
                semantic_heatmap_res,
                warning_update_res,
                structural_heatmap_res,
                structural_report_res,
                structural_export_res
            )

        # Function to interpret results using LLM
        def interpret_results(csv_path, progress=gr.Progress()):
            try:
                if not csv_path or not Path(csv_path).exists():
                    return "Please run the analysis first to generate results."
                
                # Read the CSV file
                df_results = pd.read_csv(csv_path)
                
                # Show detailed progress messages with percentages
                progress(0, desc="Preparing data for analysis...")
                progress(0.1, desc="Analyzing similarity patterns...")
                progress(0.2, desc="Connecting to Mistral 7B via OpenRouter...")
                
                # Get interpretation from LLM (using OpenRouter API)
                progress(0.3, desc="Generating scholarly interpretation (this may take 20-40 seconds)...")
                llm_service = LLMService()
                interpretation = llm_service.analyze_similarity(df_results)
                
                # Simulate completion steps
                progress(0.9, desc="Formatting results...")
                progress(0.95, desc="Applying scholarly formatting...")
                
                # Completed
                progress(1.0, desc="Analysis complete!")
                
                # Add a timestamp to the interpretation
                timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
                interpretation = f"{interpretation}\n\n<small>Analysis generated on {timestamp}</small>"
                return interpretation
            except Exception as e:
                logger.error(f"Error in interpret_results: {e}", exc_info=True)
                return f"Error interpreting results: {str(e)}"
        
        process_btn.click(
            fn=run_pipeline,
            inputs=[file_input, semantic_toggle_radio, model_dropdown, stopwords_dropdown, batch_size_slider, progress_bar_checkbox],
            outputs=[
                csv_output,
                metrics_preview,
                word_count_plot,
                heatmap_tabs["Jaccard Similarity (%)"],
                heatmap_tabs["Normalized LCS"],
                heatmap_tabs["Semantic Similarity"],
                warning_box,
                structural_heatmap,
                structural_report,
                structural_export,
            ]
        )
        
        # Connect the interpret button
        interpret_btn.click(
            fn=interpret_results,
            inputs=[csv_output],
            outputs=interpretation_output
        )
        
    return demo


if __name__ == "__main__":
    demo = main_interface()
    demo.launch()