Update app.py

app.py

@@ -12,7 +12,27 @@ def encode_text(hindi_text):
     token_ids = tokenizer.encode(hindi_text)
     return token_ids
 
-
+def encode_text_with_compression(hindi_text):
+    """
+    Encodes the given Hindi text into token IDs and calculates the compression ratio.
+    """
+    # Get token IDs
+    token_ids = tokenizer.encode(hindi_text)
+    
+    # Calculate the original text size in bytes
+    text_byte_length = len(hindi_text.encode('utf-8'))
+    
+    # Calculate the number of token IDs
+    token_id_length = len(token_ids)
+    
+    # Compression ratio
+    if text_byte_length > 0:
+        compression_ratio = token_id_length / text_byte_length
+    else:
+        compression_ratio = 0  # Handle edge case for empty input
+    
+    return token_ids, f"{compression_ratio:.2f}"
+    
 def decode_tokens(token_ids):
     """
     Decodes the given token IDs into Hindi text.
@@ -36,14 +56,15 @@ with gr.Blocks() as app:
             gr.Markdown("### Encode Hindi Text to Token IDs")
             hindi_text_input = gr.Textbox(label="Enter Hindi Text")
             token_ids_output = gr.Textbox(label="Token IDs (Encoded)", interactive=False)
+            compression_ratio_output = gr.Textbox(label="Compression Ratio", interactive=False)
             encode_button = gr.Button("Encode")
             
             # Example for encoding
             encode_example = gr.Examples(
                 examples=["मेरा भारत महान॥", "आपका घर कितनी दूर है?", "स्वतंत्रता दिवस", "द क्विक ब्राउन फॉक्स जम्प्स ओवर ए लेज़ी डॉग।"],
                 inputs=hindi_text_input,
-                outputs=token_ids_output,
-                fn=encode_text
+                outputs=[token_ids_output, compression_ratio_output],
+                fn=encode_text_with_compression
             )
         
         with gr.Column():
@@ -54,7 +75,11 @@ with gr.Blocks() as app:
             
             
     
-    encode_button.click(encode_text, inputs=hindi_text_input, outputs=token_ids_output)
+    encode_button.click(
+        encode_text_with_compression, 
+        inputs=hindi_text_input, 
+        outputs=[token_ids_output, compression_ratio_output]
+    )
     decode_button.click(decode_tokens, inputs=token_ids_input, outputs=decoded_text_output)
 
 app.launch()