stream on full warc

app.py

@@ -26,8 +26,6 @@ from datatrove.utils.typeshelper import Languages
 
 nltk.download('punkt_tab')
 DUMP_TO_PROCESS = "CC-MAIN-2023-50"
-default_output_docs_2k = pd.read_json(f"output_all-2k/base_processing/output/{DUMP_TO_PROCESS}/00000.jsonl.gz", compression="gzip", lines=True).to_dict(orient="records")
-default_output_docs_200 = pd.read_json(f"output_all-200/base_processing/output/{DUMP_TO_PROCESS}/00000.jsonl.gz", compression="gzip", lines=True).to_dict(orient="records")
 
 make_gallery_image_buttons_js = """
 function load() {
@@ -81,12 +79,6 @@ function load() {
 }
 """
 css = """
-tr:has(> td div span span div.diffInsertion) {
-    background: darkgreen;
-}
-tr:has(> td div span span div.diffDeletion) {
-    background: darkred;
-}
 tr td {
     border-top: 1px solid black;
 }
@@ -399,9 +391,10 @@ with gr.Blocks(css=css, js=make_gallery_image_buttons_js) as demo:
             
             pipeline_executor = LocalPipelineExecutor(
                 pipeline=[
-                    JsonlReader(data_folder=f"output_text_extraction-2k/base_processing/output/{DUMP_TO_PROCESS}", glob_pattern="*.jsonl.gz"),
+                    JsonlReader(data_folder=f"output_text_extraction-full/base_processing/output/{DUMP_TO_PROCESS}", glob_pattern="*.jsonl.gz"),
                     partial(increment_num_warc_samples, num_warc_samples_per_doc=2000 / 1687)
                 ] + steps_to_run[2:] + [
+                    lambda data, rank, world_size: islice(data, 100),
                     lambda data, rank, world_size: map(output_docs.append, data)
                 ],
                 logging_dir="logs",
@@ -411,8 +404,9 @@ with gr.Blocks(css=css, js=make_gallery_image_buttons_js) as demo:
             pipeline_executor = LocalPipelineExecutor(
                 pipeline=[
                     WarcReader(data_folder="data", glob_pattern="*.warc.gz"),
-                    lambda data, rank, world_size: islice(data, num_warc_samples),
+                    increment_num_warc_samples
                 ] + steps_to_run + [
+                    lambda data, rank, world_size: islice(data, 100),
                     lambda data, rank, world_size: map(output_docs.append, data)
                 ],
                 logging_dir="logs",
@@ -445,12 +439,12 @@ with gr.Blocks(css=css, js=make_gallery_image_buttons_js) as demo:
                     output_tab: gr.Tab("Output (loading...)"),
                     excluded_tab: gr.Tab("Excluded (loading...)"),
                     **{
-                        excluded_dataframes[type(step_to_run)]: pd.DataFrame({"text": [doc.text for doc in step_to_run.exclusion_writer.docs]})
+                        excluded_dataframes[type(step_to_run)]: pd.DataFrame({"text": []})
                         for step_to_run in pipeline_executor.pipeline
                         if isinstance(step_to_run, BaseFilter) and type(step_to_run) in excluded_dataframes
                     },
                     **{
-                        excluded_tabs[type(step_to_run)]: gr.Tab(f"{type(step_to_run).__name__} (~{len(step_to_run.exclusion_writer.docs)/num_warc_samples*100:.03f}% of data)")
+                        excluded_tabs[type(step_to_run)]: gr.Tab(f"{type(step_to_run).__name__}")
                         for step_to_run in pipeline_executor.pipeline
                         if isinstance(step_to_run, BaseFilter) and type(step_to_run) in excluded_dataframes
                     },
@@ -470,6 +464,8 @@ with gr.Blocks(css=css, js=make_gallery_image_buttons_js) as demo:
                 if isinstance(step_to_run, BaseFilter) and type(step_to_run) in excluded_dataframes
             },
         }
+    
+    stop_button.click(cancels=[view_pipeline_results])
 
 if __name__ == "__main__":
     demo.launch()

output_all-2k/base_processing/output/CC-MAIN-2023-50/00000.jsonl.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e32e8d71f3bcf050fe2d0a744b21e411c66b95ee4f88a4fa8eac5f6b459ce345
-size 299465

{output_all-200 → output_text_extraction-full}/base_processing/output/CC-MAIN-2023-50/00000.jsonl.gz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7798c4f4cb22b19532ea4c5a58dbc94b72dd7627007a90dc0b2d83cf6ac56007
-size 28010
+oid sha256:b7d0b9e9dd069a639f56c3ad81e92bdb053e983899cb7910f633af174f44903d
+size 28830205