Update ASCARIS.py

ASCARIS.py

@@ -11,10 +11,37 @@ import argparse
 from st_aggrid import AgGrid, GridOptionsBuilder, JsCode,GridUpdateMode
 import base64
 showWarningOnDirectExecution = False
-from datasets import load_dataset
-alphafold_path = load_dataset("HUBioDataLab/AlphafoldStructures")
-st.write('path')
-st.write(alphafold_path)
+
+
+from datasets import Dataset, concatenate_datasets
+MAX_SAMPLES_IN_MEMORY = 1000
+samples_in_dset = 0
+dset = Dataset.from_dict({"col1": [], "col2": []})  # empty dataset
+path_to_save_dir = "HUBioData/input_files"
+num_chunks = 0
+for example_dict in custom_example_dict_streamer("HUBioData/AlphafoldStructures"):
+    dset = dset.add_item(example_dict)
+    samples_in_dset += 1
+    if samples_in_dset == MAX_SAMPLES_IN_MEMORY:
+        samples_in_dset = 0
+        dset.save_to_disk(f"{path_to_save_dir}{num_chunks}")
+        num_chunks =+ 1
+        dset = Dataset.from_dict({"col1": [], "col2": []})  # empty dataset
+if samples_in_dset > 0:
+    dset.save_to_disk(f"{path_to_save_dir}{num_chunks}")
+    num_chunks =+ 1
+loaded_dsets = []  # memory-mapped
+for chunk_num in range(num_chunks):
+    dset = Dataset.load_from_disk(f"{path_to_save_dir}{chunk_num}") 
+    loaded_dsets.append(dset)
+final_dset = concatenate_datasets(dset)
+st.write('FİNAL DSET')
+st.write(final_dset)
+
+
+
+
+
 def convert_df(df):
    return df.to_csv(index=False).encode('utf-8')