{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[],"toc_visible":true},"widgets":{"application/vnd.jupyter.widget-state+json":{"9dd744f6393943f2959c27ac89e93ba1":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_27b05fd0ecb34c10ae93eac9d4cdc1c5","IPY_MODEL_025ddae40b4541a684f1d76c5b289ab6","IPY_MODEL_bd4b2fd97cb34f5f900e91969cc588c7"],"layout":"IPY_MODEL_48641e4f73954da987f034a74c3b8b5e"}},"27b05fd0ecb34c10ae93eac9d4cdc1c5":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_c8177522a2e64f4eafef0d6cbc1f6cdd","placeholder":"​","style":"IPY_MODEL_29a4704b56cf4d4fa145fac101dbcb8d","value":"TrainGemma2.gameintro.queries.lf.csv: 100%"}},"025ddae40b4541a684f1d76c5b289ab6":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_c345a572c5d4493d93423618f0a6f143","max":32325930,"min":0,"orientation":"horizontal","style":"IPY_MODEL_813942f77ed3426a8a6261ac60542f48","value":32325930}},"bd4b2fd97cb34f5f900e91969cc588c7":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_e35595cc50a24fcca6f812997c8c8dc5","placeholder":"​","style":"IPY_MODEL_63175136102c4135acce3e700bf92cf8","value":" 32.3M/32.3M [00:00&lt;00:00, 34.5MB/s]"}},"48641e4f73954da987f034a74c3b8b5e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c8177522a2e64f4eafef0d6cbc1f6cdd":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"29a4704b56cf4d4fa145fac101dbcb8d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"c345a572c5d4493d93423618f0a6f143":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"813942f77ed3426a8a6261ac60542f48":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"e35595cc50a24fcca6f812997c8c8dc5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"63175136102c4135acce3e700bf92cf8":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"528f68abd3af4761b1b00b40599e0fbf":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_f84cfa74576a46e88caa9989161d08fc","IPY_MODEL_5d03f9e044f146b7b36b34cdcf21153a","IPY_MODEL_498c133792ea4cd38b6385be5f7f8c62"],"layout":"IPY_MODEL_141a311668984b2ebaca90db0c57d815"}},"f84cfa74576a46e88caa9989161d08fc":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_e04c52ea5941431f9d1dd12d2f41654a","placeholder":"​","style":"IPY_MODEL_27daab81e7b64087abae1b5e2a3c0c5d","value":"Generating train split: "}},"5d03f9e044f146b7b36b34cdcf21153a":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_14b2db86f938477696e3e54b321caeed","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_fdb857f832a54286ba504046a177a883","value":1}},"498c133792ea4cd38b6385be5f7f8c62":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_ec1882dbce7d46e28580b8cf2b4da19b","placeholder":"​","style":"IPY_MODEL_860d1a845d764ff6853ae56e6ac3afcc","value":" 28599/0 [00:00&lt;00:00, 35222.06 examples/s]"}},"141a311668984b2ebaca90db0c57d815":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e04c52ea5941431f9d1dd12d2f41654a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"27daab81e7b64087abae1b5e2a3c0c5d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"14b2db86f938477696e3e54b321caeed":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"20px"}},"fdb857f832a54286ba504046a177a883":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"ec1882dbce7d46e28580b8cf2b4da19b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"860d1a845d764ff6853ae56e6ac3afcc":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"b43311c7e443439d9776c1cb51f07baf":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_c121083f404b494f881cab7d9ec122d6","IPY_MODEL_729a4366db2d4203adb0fd2b0fb2dd4c","IPY_MODEL_96b6572ec6184bfcb655ba62329631ec"],"layout":"IPY_MODEL_ca83618081ae48ee88976c512c3f2022"}},"c121083f404b494f881cab7d9ec122d6":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_5f4a077a96ad446cb41241108eac2618","placeholder":"​","style":"IPY_MODEL_4a562f7c91b849599ff06f76da226c85","value":"Map: 100%"}},"729a4366db2d4203adb0fd2b0fb2dd4c":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_3a7e8f584e72412c8fa4fd8063630e2b","max":28599,"min":0,"orientation":"horizontal","style":"IPY_MODEL_39d9dd8bc753487f9a50c939b9ed38e5","value":28599}},"96b6572ec6184bfcb655ba62329631ec":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_523070898a474b74bde2d365c69dddff","placeholder":"​","style":"IPY_MODEL_fced41bbaf3a496aa62080a6b88e1afc","value":" 28599/28599 [00:00&lt;00:00, 50440.48 examples/s]"}},"ca83618081ae48ee88976c512c3f2022":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5f4a077a96ad446cb41241108eac2618":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4a562f7c91b849599ff06f76da226c85":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"3a7e8f584e72412c8fa4fd8063630e2b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"39d9dd8bc753487f9a50c939b9ed38e5":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"523070898a474b74bde2d365c69dddff":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"fced41bbaf3a496aa62080a6b88e1afc":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":11293954,"sourceType":"datasetVersion","datasetId":7061846}],"dockerImageVersionId":30919,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Training on Your Private Data - by NathMath @ bilibili","metadata":{"id":"Wj4lhJCuMYcm"}},{"cell_type":"markdown","source":"大家好，这里是Nathmath。我将用一个系列给你们讲解如何基于预训练好的底模进行大语言模型的私有数据微调。\n> * 区别于部分UP主，我可能会“废话”很多。但是，“废话”才是你们学习的基础。因为我的“废话”是在讲解原理，让你们`能“鱼”也能“渔”`（钓鱼），意思是懂得原理，就可以不仅仅学会这一个微调，并且能够自己用在其他需要的地方，迁移学习。而不是仅仅学会我这一个东西，无法举一反三。\n\n> * 本系列视频特别推荐大家动手。以本期视频举例，很多同学还不会准备数据集，没事，请一定要拿我的数据先跑一遍，遍跑遍听我的讲解，理解每一步在做什么；我后面的视频会继续教你们怎么准备数据集（会的同学仅看本期就可以），以及怎么进行多轮对话训练、怎么进行思考训练、怎么进行其他模型的训练；当然，最基础的，建议大家自己`先照猫画虎把我的Notebook跑通`，然后再自己尝试自己的数据。\n\n> * 微调和训练是很难很难的内容。包括训练数据准备。在行内，有着“`数据处理80%，建模训练20%`”的行话，意思是数据处理所消耗的时间和精力占到整个机器学习的80%，其也决定了你模型的质量的80%，因为\"garbage in, garbage out\"（进去的是垃圾，出来的也是垃圾）。大家`一定不要灰心`，如果想学的话，踏踏实实学，有问题就问ChatGPT/DeepSeek，它能解决很多问题。\n\n> * 关于在线训练平台。UP个人推荐Kaggle。原因是`每周`有30小时的免费的T4(16G)x2的GPU使用，需要注册并完成手机号认证(认证时候中国手机记着加上86)。另外提醒，数据特别敏感的个人或者企业用户请自己花钱租用服务器。","metadata":{}},{"cell_type":"markdown","source":"v20 版本更新的内容：\n* 使用了冷启动，将ckpt-1418的模型导入并继续训练\n* 新增了基于同内容的多轮对话生成，改善了多轮对话生成不同游戏内容的能力，防止过拟合。\n* 继续训练了基于多轮对话的训练集，防止了过拟合","metadata":{}},{"cell_type":"markdown","source":"## 1. Prepare the Environment","metadata":{"id":"IlrY86-MNfjf"}},{"cell_type":"code","source":"# Reference https://colab.research.google.com/drive/1Ys44kVvmeZtnICzWz0xgpRnrIOjZAuxp?usp=sharing#scrollTo=FqfebeAdT073\n# 参考文献","metadata":{"id":"0AhWNVC9U9B4","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* Unsloth supports Llama, Mistral, Phi-3, Gemma, Yi, DeepSeek, Qwen, TinyLlama, Vicuna, Open Hermes etc\n* Unsloth supports 16bit LoRA or 4bit QLoRA. Both 2x faster.\n* With [PR 26037](https://github.com/huggingface/transformers/pull/26037), we support downloading 4bit models **4x faster**! [Our repo](https://huggingface.co/unsloth) has Llama, Mistral 4bit models.","metadata":{"id":"8l_HaJMosoVY"}},{"cell_type":"code","source":"# Modified Auther NathMath, open-sourced with Apache-2.0 Licence\n# 修改作者：NathMath，以Apache-2.0 Licence许可证开源","metadata":{"id":"7FXuUqc9j1dw","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use Multi-GPUs if available\n# 可行时使用双CPU，适用于Kaggle T4x2\n\nimport os\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1\"\nprint(\"Completed\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:26:01.660290Z","iopub.execute_input":"2025-04-08T05:26:01.660650Z","iopub.status.idle":"2025-04-08T05:26:01.665503Z","shell.execute_reply.started":"2025-04-08T05:26:01.660622Z","shell.execute_reply":"2025-04-08T05:26:01.664768Z"}},"outputs":[{"name":"stdout","text":"Completed\n","output_type":"stream"}],"execution_count":1},{"cell_type":"code","source":"# Install or import unsloth\n# 安装或导入用于微调的unsloth库\nif \"Mute\" != None:\n    # 静音安装\n    !pip install unsloth==\"2025.3.18\" > /dev/null\nelse:\n    !pip install unsloth==\"2025.3.18\"\n\n# It is slow; so be patient\n# 这一步很慢请耐心等待","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:26:11.535996Z","iopub.execute_input":"2025-04-08T05:26:11.536307Z","iopub.status.idle":"2025-04-08T05:30:27.136968Z","shell.execute_reply.started":"2025-04-08T05:26:11.536242Z","shell.execute_reply":"2025-04-08T05:30:27.135901Z"}},"outputs":[{"name":"stdout","text":"\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\nfastai 2.7.18 requires torch<2.6,>=1.10, but you have torch 2.6.0 which is incompatible.\nlangchain 0.3.12 requires async-timeout<5.0.0,>=4.0.0; python_version < \"3.11\", but you have async-timeout 5.0.1 which is incompatible.\npylibcugraph-cu12 24.10.0 requires pylibraft-cu12==24.10.*, but you have pylibraft-cu12 25.2.0 which is incompatible.\npylibcugraph-cu12 24.10.0 requires rmm-cu12==24.10.*, but you have rmm-cu12 25.2.0 which is incompatible.\ntensorflow-decision-forests 1.10.0 requires tensorflow==2.17.0, but you have tensorflow 2.17.1 which is incompatible.\ntorchaudio 2.5.1+cu121 requires torch==2.5.1, but you have torch 2.6.0 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0m","output_type":"stream"}],"execution_count":3},{"cell_type":"code","source":"# DO NOT CARE BUG \"ERROR: pip's dependency resolver does not currently take into account\"\n# 这个报错不用管：“ERROR: pip's dependency resolver does not currently take into account”","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import torch backend\n# 导入torch后端\nimport torch\n\ntorch_version = torch.cuda.get_device_capability()\ntorch_major_v, torch_minor_v = torch_version\nprint(torch_major_v, torch_minor_v)\n# The first version digit must be greater or equal to 7, or a bug will be raised\n# 第一个数大版本必须为7或者以上，否则会提示CUDA运算版本不足bug\n\n# If an error is thrown here, then it means you DO NOT have a valid NVIDIA accelerator\n# 如果这里报错，那么意味着你没有一个有效的NVIDIA显卡作为运算加速器，请选择T4x2而不是P100，P100会提示版本不足","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:41:49.612616Z","iopub.execute_input":"2025-04-08T05:41:49.612908Z","iopub.status.idle":"2025-04-08T05:41:51.695290Z","shell.execute_reply.started":"2025-04-08T05:41:49.612886Z","shell.execute_reply":"2025-04-08T05:41:51.694559Z"}},"outputs":[{"name":"stdout","text":"7 5\n","output_type":"stream"}],"execution_count":4},{"cell_type":"code","source":"# Install other dependences\n# 安装其他依赖项\nif \"Mute\" != None:\n    # 静音安装\n    !pip install xformers trl peft accelerate bitsandbytes  > /dev/null\nelse:\n    !pip install xformers trl peft accelerate bitsandbytes ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:41:55.651899Z","iopub.execute_input":"2025-04-08T05:41:55.652331Z","iopub.status.idle":"2025-04-08T05:42:03.348910Z","shell.execute_reply.started":"2025-04-08T05:41:55.652304Z","shell.execute_reply":"2025-04-08T05:42:03.347897Z"}},"outputs":[],"execution_count":5},{"cell_type":"code","source":"# Import unsloth FastLanguageModel\n# 导入FastLanguageModel\nfrom unsloth import FastLanguageModel","metadata":{"id":"1IGsxSprNG63","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:42:46.085651Z","iopub.execute_input":"2025-04-08T05:42:46.085959Z","iopub.status.idle":"2025-04-08T05:43:25.929876Z","shell.execute_reply.started":"2025-04-08T05:42:46.085936Z","shell.execute_reply":"2025-04-08T05:43:25.929192Z"}},"outputs":[{"name":"stdout","text":"🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\nUnsloth: Failed to patch Gemma3ForConditionalGeneration.\n🦥 Unsloth Zoo will now patch everything to make training faster!\n","output_type":"stream"}],"execution_count":6},{"cell_type":"code","source":"# See if both GPUs are activated\n# 看看是否两个GPU都被激活了\n\ngpu_count = torch.cuda.device_count()\nprint(\"GPU Number:\", gpu_count)\nfor i in range(gpu_count):\n    print(f\"GPU {i}: {torch.cuda.get_device_name(i)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:43:59.860873Z","iopub.execute_input":"2025-04-08T05:43:59.861310Z","iopub.status.idle":"2025-04-08T05:43:59.867419Z","shell.execute_reply.started":"2025-04-08T05:43:59.861253Z","shell.execute_reply":"2025-04-08T05:43:59.866618Z"}},"outputs":[{"name":"stdout","text":"GPU Number: 2\nGPU 0: Tesla T4\nGPU 1: Tesla T4\n","output_type":"stream"}],"execution_count":7},{"cell_type":"code","source":"# Import training utilities\n# 导入其他训练工具\nfrom trl import SFTTrainer\nfrom transformers import TrainingArguments\nfrom unsloth import is_bfloat16_supported","metadata":{"id":"vrYjQLxTSFjN","colab":{"base_uri":"https://localhost:8080/","height":387},"outputId":"ce5ca1de-43d8-414b-b72f-8f811d7e42cf","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:44:01.919847Z","iopub.execute_input":"2025-04-08T05:44:01.920172Z","iopub.status.idle":"2025-04-08T05:44:01.924196Z","shell.execute_reply.started":"2025-04-08T05:44:01.920132Z","shell.execute_reply":"2025-04-08T05:44:01.923332Z"}},"outputs":[],"execution_count":8},{"cell_type":"code","source":"# Import data science packeges\n# 导入数据科学使用的包\nimport random\nimport numpy as np\nimport pandas as pd","metadata":{"id":"GLUb83gYSxMW","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:44:04.863784Z","iopub.execute_input":"2025-04-08T05:44:04.864063Z","iopub.status.idle":"2025-04-08T05:44:04.867713Z","shell.execute_reply.started":"2025-04-08T05:44:04.864041Z","shell.execute_reply":"2025-04-08T05:44:04.866917Z"}},"outputs":[],"execution_count":9},{"cell_type":"code","source":"# By Nathmath","metadata":{"id":"iaZJQxXascfv","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:44:07.966416Z","iopub.execute_input":"2025-04-08T05:44:07.966688Z","iopub.status.idle":"2025-04-08T05:44:07.970427Z","shell.execute_reply.started":"2025-04-08T05:44:07.966668Z","shell.execute_reply":"2025-04-08T05:44:07.969542Z"}},"outputs":[],"execution_count":10},{"cell_type":"markdown","source":"## 2. Configurate the underlying model","metadata":{"id":"YK_VnKgONnIe"}},{"cell_type":"code","source":"# HF token\n# HF 的token，如果你需要把训练好的模型保存到hugging face时需要\n_global_hf_token = \"\"","metadata":{"id":"Gm712pctXX3V","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:44:10.811668Z","iopub.execute_input":"2025-04-08T05:44:10.811951Z","iopub.status.idle":"2025-04-08T05:44:10.815570Z","shell.execute_reply.started":"2025-04-08T05:44:10.811931Z","shell.execute_reply":"2025-04-08T05:44:10.814722Z"}},"outputs":[],"execution_count":11},{"cell_type":"code","source":"# Model configuration\n# 模型设定\n_global_model_name = \"unsloth/gemma-2-9b-bnb-4bit\"    # HF 模型识别名称\n_global_model_max_seqlen = 4096                       # 模型的最长输出tokens数，小说设置到8192，但显著增加训练时间\n                                                      # v-20 更改到4096, 原2048\n_global_model_dtype = None\n_global_model_load_in_4bit = True\n","metadata":{"id":"PMXPujtPN0I1","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:44:29.826804Z","iopub.execute_input":"2025-04-08T05:44:29.827125Z","iopub.status.idle":"2025-04-08T05:44:29.830719Z","shell.execute_reply.started":"2025-04-08T05:44:29.827100Z","shell.execute_reply":"2025-04-08T05:44:29.829986Z"}},"outputs":[],"execution_count":12},{"cell_type":"code","source":"# Model training seed configuration\n# 模型训练时的种子，随机生成一个，你也可以自己设定一个\n_train_seed = int(np.random.rand() * 2 ** 32)\n_train_seed","metadata":{"id":"epDSH2s4Sosn","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:44:33.255059Z","iopub.execute_input":"2025-04-08T05:44:33.255407Z","iopub.status.idle":"2025-04-08T05:44:33.266474Z","shell.execute_reply.started":"2025-04-08T05:44:33.255376Z","shell.execute_reply":"2025-04-08T05:44:33.265704Z"}},"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":"4127228089"},"metadata":{}}],"execution_count":13},{"cell_type":"markdown","source":"## 2.1. Load the base model into the environment","metadata":{}},{"cell_type":"code","source":"# Load the pretrained model\n# 导入HF上预训练好的底模\nif False:\n    _global_model, _global_tokenizer = FastLanguageModel.from_pretrained(\n    model_name=_global_model_name,\n    max_seq_length=_global_model_max_seqlen,\n    dtype = _global_model_dtype,\n    load_in_4bit = _global_model_load_in_4bit\n)\nelse:\n    _global_model, _global_tokenizer = FastLanguageModel.from_pretrained(\n    model_name=\"DOFOFFICIAL/ThisIsNewTmp\",  # Your previously saved model\n    max_seq_length=_global_model_max_seqlen,\n    dtype = _global_model_dtype,\n    load_in_4bit = _global_model_load_in_4bit\n)\n# You must see Tesla T4. Num GPUs = 2. here, or you only have 1 GPU\n# 在这里你必须看到GPU数量=2否则你只有1个GPU，训练会慢很多","metadata":{"id":"HzK6KLZUSnml","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:45:26.258122Z","iopub.execute_input":"2025-04-08T05:45:26.258621Z","iopub.status.idle":"2025-04-08T05:45:58.091371Z","shell.execute_reply.started":"2025-04-08T05:45:26.258592Z","shell.execute_reply":"2025-04-08T05:45:58.090682Z"}},"outputs":[{"name":"stdout","text":"==((====))==  Unsloth 2025.3.18: Fast Gemma2 patching. Transformers: 4.51.0.\n   \\\\   /|    Tesla T4. Num GPUs = 2. Max memory: 14.741 GB. Platform: Linux.\nO^O/ \\_/ \\    Torch: 2.6.0+cu124. CUDA: 7.5. CUDA Toolkit: 12.4. Triton: 3.2.0\n\\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.29.post3. FA2 = False]\n \"-____-\"     Free license: http://github.com/unslothai/unsloth\nUnsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"model.safetensors:   0%|          | 0.00/6.13G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6c000ff15aaf46c0a656f560f9b4fd6b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"generation_config.json:   0%|          | 0.00/190 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"82b6faa4fcb24a1bb44a66c77a8f691e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json:   0%|          | 0.00/46.4k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d0f1377adbfb46dda813cb2d7d98865d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer.model:   0%|          | 0.00/4.24M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"028d4e78d4c94b5ba4edbf7eaf0fcc78"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"special_tokens_map.json:   0%|          | 0.00/636 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"32614765bdc44161bed6eb62c534b3d3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer.json:   0%|          | 0.00/17.5M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"22398cc24ddd40a5a445ae83a2405fde"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"adapter_model.safetensors:   0%|          | 0.00/432M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4dd023cd4d5740a69aeefb0f43c989ed"}},"metadata":{}},{"name":"stderr","text":"Unsloth 2025.3.18 patched 42 layers with 42 QKV layers, 42 O layers and 42 MLP layers.\n","output_type":"stream"}],"execution_count":14},{"cell_type":"markdown","source":"## 3. Load and process the data used","metadata":{}},{"cell_type":"code","source":"# Patch the model and enable LoRA adapters\n# 设置模型补丁：你要怎么选择训练参数\n\n# Only run this block when you FIRST configurate your model\n# 只需要初始化一次，在最开始patch你的模型的时候跑就可以\nif False:\n    _global_model = FastLanguageModel.get_peft_model(\n        _global_model,\n        r = 32,                 # 选择任何大于0 的数，建议16或32或64，过大容易过拟合，且会很慢\n        target_modules = [\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\", \"gate_proj\", \"up_proj\", \"down_proj\",],\n        lora_alpha = 16,        # 同上，建议选择16或者32\n        lora_dropout = 0,       # 建议选择0加速，或者0.1略微进行正则化防止过拟合\n        bias = \"none\",\n        use_gradient_checkpointing = \"unsloth\",\n        random_state = _train_seed,\n        use_rslora = False,     # True以开启rank stabilized LoRA，建议选False\n        loftq_config = None,\n        # Unsloth有LoRA adapters技术，因此不用训练整个模型，仅仅训练1~10%的层左右\n    )","metadata":{"id":"KV1C0R0lPu5P","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Standard Alpaca Prompt formatting template\n# 标准Alpaca提示词模板\nalpaca_prompt = \"\"\"{}\n\n### Instruction:\n{}\n\n### Response:\n{}\n\n\"\"\"\nalpaca_prompt_multi = \"\"\"### Instruction:\n{}\n\n### Response:\n{}\n\n\"\"\"\nalpaca_prompt_infer = \"\"\"{}\n\n### Instruction:\n{}\n\n### Response:\n\"\"\"\n# {}0: System Prompt 系统提示词\n# {}1: User Prompt 用户提示词\n# {}2: Assistant Response 智能助理回复\n\n# Must add EOS_TOKEN\nEOS_TOKEN = _global_tokenizer.eos_token\n","metadata":{"id":"BkpvoEwDQzFN","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:46:36.834639Z","iopub.execute_input":"2025-04-08T05:46:36.834922Z","iopub.status.idle":"2025-04-08T05:46:36.839111Z","shell.execute_reply.started":"2025-04-08T05:46:36.834901Z","shell.execute_reply":"2025-04-08T05:46:36.838190Z"}},"outputs":[],"execution_count":16},{"cell_type":"code","source":"# Formatting prompts for training\ndef formatting_prompts(query):\n    system    = query[\"system\"]\n    instructions = query[\"user\"]\n    outputs   = query[\"assistant\"]\n    texts = []\n    for syst, usr, assit in zip(system, instructions, outputs):\n        # Process \\n\\n to \\n to avoid alpaca issues\n        syst = syst.replace(\"\\n\\n\", \"\\n\") if syst is not None else \"\"\n        usr = usr.replace(\"\\n\\n\", \"\\n\")\n        assit = assit.replace(\"\\n\\n\", \"\\n\")\n\n        # Must add EOS_TOKEN, otherwise your generation will go on forever!\n        text = alpaca_prompt.format(syst, usr, assit) + EOS_TOKEN\n        texts.append(text)\n    return { \"formatted\" : texts, }\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:46:40.147788Z","iopub.execute_input":"2025-04-08T05:46:40.148074Z","iopub.status.idle":"2025-04-08T05:46:40.153063Z","shell.execute_reply.started":"2025-04-08T05:46:40.148053Z","shell.execute_reply":"2025-04-08T05:46:40.152336Z"}},"outputs":[],"execution_count":17},{"cell_type":"code","source":"# Formatting prompts for multi-round training\ndef formatting_prompts_multi(query, \n                              *, \n                             m: int = 2, \n                             n: int = 12, \n                             num_conversations: int | None | float = None,\n                             seed : int | None = None):\n    \"\"\"\n    Constructs formatted text for multi-round conversations.\n    Simply converted. Assume conversations are independent\n    \n    Parameters:\n    query: dict, containing three keys \"system\", \"user\", and \"assistant\", corresponding to a list of single-round conversation data.\n    m, n: int, specifies the interval [m, n] for randomly selecting the number of conversation rounds.\n    num_conversations: The number of multi-round conversations generated, which is the same as the number of original data by default.\n    seed: int | None, optional, used to set the random seed to ensure reproducibility.\n    \"\"\"\n    system = query[\"system\"]\n    instructions = query[\"user\"]\n    outputs = query[\"assistant\"]\n\n    if seed is not None:\n        random.seed(seed)\n\n    total_samples = len(system)\n    # If the number of generated samples is not specified, \n    # the default is to generate multiple rounds of dialogue with the same number of original samples.\n    if num_conversations is None:\n        num_conversations = total_samples\n    elif isinstance(num_conversations, float):\n        num_conversations = int(total_samples * num_conversations)\n\n    texts = []\n    for _ in range(num_conversations):\n        # Randomly generate the number of rounds t included in this multi-round dialogue \n        # (not exceeding the total number of samples)\n        t = random.randint(m, min(n, total_samples))\n       \n        # Randomly draw t samples of index (no repetition)\n        indices = random.sample(range(total_samples), t)\n       \n        # Use the first sampled system prompt as the start of the conversation\n        conv_system = system[indices[0]]\n        conv_system = conv_system.replace(\"\\n\\n\", \"\\n\") if conv_system is not None else \"\"\n        conv_text = \"\"\n        # Traverse the extracted samples and add user input and \n        # assistant replies in sequence\n        for i, idx in enumerate(indices):\n            usr = instructions[idx].replace(\"\\n\\n\", \"\\n\")\n            assit = outputs[idx].replace(\"\\n\\n\", \"\\n\")\n            # Add user + assist conditional on whether it is the 1st or not\n            if i == 0:\n                conv_text += alpaca_prompt.format(conv_system, usr, assit)\n            else:\n                conv_text += alpaca_prompt_multi.format(usr, assit)\n            # Must add EOS_TOKEN, otherwise your generation will go on forever!\n            conv_text += EOS_TOKEN\n            \n        texts.append(conv_text)\n    return {\"formatted\": texts, }\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:46:41.932447Z","iopub.execute_input":"2025-04-08T05:46:41.932729Z","iopub.status.idle":"2025-04-08T05:46:41.939982Z","shell.execute_reply.started":"2025-04-08T05:46:41.932709Z","shell.execute_reply":"2025-04-08T05:46:41.939338Z"}},"outputs":[],"execution_count":18},{"cell_type":"code","source":"# Load your local dataset\n# 加载你的dataset(替换成你的csv文件)\nfrom datasets import load_dataset\n\n# Replace this by your remote file\n# 替换这个链接为你的github/huggingface数据文件，建议使用csv/parq\nurl = r\"https://huggingface.co/DOFOFFICIAL/NathUI-Tutorial/resolve/main/dataset/TrainGemma2.gameintro.queries.v2.lf.csv\"\ndataset = load_dataset(\"csv\", data_files=url, split = \"train\")","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":81,"referenced_widgets":["9dd744f6393943f2959c27ac89e93ba1","27b05fd0ecb34c10ae93eac9d4cdc1c5","025ddae40b4541a684f1d76c5b289ab6","bd4b2fd97cb34f5f900e91969cc588c7","48641e4f73954da987f034a74c3b8b5e","c8177522a2e64f4eafef0d6cbc1f6cdd","29a4704b56cf4d4fa145fac101dbcb8d","c345a572c5d4493d93423618f0a6f143","813942f77ed3426a8a6261ac60542f48","e35595cc50a24fcca6f812997c8c8dc5","63175136102c4135acce3e700bf92cf8","528f68abd3af4761b1b00b40599e0fbf","f84cfa74576a46e88caa9989161d08fc","5d03f9e044f146b7b36b34cdcf21153a","498c133792ea4cd38b6385be5f7f8c62","141a311668984b2ebaca90db0c57d815","e04c52ea5941431f9d1dd12d2f41654a","27daab81e7b64087abae1b5e2a3c0c5d","14b2db86f938477696e3e54b321caeed","fdb857f832a54286ba504046a177a883","ec1882dbce7d46e28580b8cf2b4da19b","860d1a845d764ff6853ae56e6ac3afcc"]},"id":"bfu7l5DuQsBA","outputId":"c40c8b78-b8d9-4f11-da05-bc1e7405bc3c","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:46:46.427604Z","iopub.execute_input":"2025-04-08T05:46:46.427893Z","iopub.status.idle":"2025-04-08T05:46:48.006494Z","shell.execute_reply.started":"2025-04-08T05:46:46.427872Z","shell.execute_reply":"2025-04-08T05:46:48.005835Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"TrainGemma2.gameintro.queries.v2.lf.csv:   0%|          | 0.00/30.7M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"944dc6d6c3ef4c8899b0b2b7732ac575"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating train split: 0 examples [00:00, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9b3255c37bbe446ba0b4c193ccc5b50c"}},"metadata":{}}],"execution_count":19},{"cell_type":"code","source":"# See what is it like\n# 看看加载的数据是否正确\ndataset[495]","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"uv1BdtwkxdX5","outputId":"7cd736a9-3914-47dd-f3a3-818bb91390ad","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:46:50.222200Z","iopub.execute_input":"2025-04-08T05:46:50.222543Z","iopub.status.idle":"2025-04-08T05:46:50.234178Z","shell.execute_reply.started":"2025-04-08T05:46:50.222519Z","shell.execute_reply":"2025-04-08T05:46:50.233285Z"}},"outputs":[{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"{'Unnamed: 0': '173_0',\n 'system': '你是一个游戏剧情规划师。',\n 'user': '请根据游戏名编写游戏介绍：【游戏名】：Marguerite Sphere -マーガレット スフィア-。',\n 'assistant': '真要用最简洁语言来形容这城镇的话，那就是『花之街』。\\n这是充满异国情调的港口城市——白美根。\\n在这所到之处皆是野生雏菊的城镇里，有着一个传说。\\n——在拥有纯粹专一思念的少年少女面前，花之精灵将为他们送上，能够实现愿望的雏菊之花——\\n如此的白美根街住着一对双胞胎兄妹——此花大地、此花茉莉，如今他们都在私立白美根学园上学。\\n妹妹成绩优秀、头脑清晰，还有着能让学校男生们都为之着迷的可爱容貌。\\n可惜的是，与那容貌相对的，她却是不管对方是谁都尖酸刻薄相待的冰山利刃。\\n也正是这巨大反差，让她成为了备受他人注目的存在。\\n然而，其实她只是异常腼腆，以至于没能好好表现自己，着实楚楚可怜的妹妹罢了。\\n这样的她和哥哥大地，就在朋友们的簇拥下过着轻松明快的每一天。\\n那样的日子，想必会一直持续下去。没错，一直持续，直到今天……\\n「你还真是百年难得一遇的女难相呀……」\\n这些话出自于一位无意间碰到却被硬拉着占卜的占卜师之口。\\n在这点缀着纯白花朵的城镇里，如梦似幻又稍稍不可思议的恋爱故事就此拉开序幕……'}"},"metadata":{}}],"execution_count":20},{"cell_type":"code","source":"# Map the training data into the formatted strings\n# 将你的原始数据转化为Alpaca格式化数据\nif False:\n    dataset_format = dataset.map(formatting_prompts, batched = True,)\nelse:\n    # Multi-round conversation 多轮对话\n    dataset_format = dataset.map(formatting_prompts_multi, batched = True)","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":49,"referenced_widgets":["b43311c7e443439d9776c1cb51f07baf","c121083f404b494f881cab7d9ec122d6","729a4366db2d4203adb0fd2b0fb2dd4c","96b6572ec6184bfcb655ba62329631ec","ca83618081ae48ee88976c512c3f2022","5f4a077a96ad446cb41241108eac2618","4a562f7c91b849599ff06f76da226c85","3a7e8f584e72412c8fa4fd8063630e2b","39d9dd8bc753487f9a50c939b9ed38e5","523070898a474b74bde2d365c69dddff","fced41bbaf3a496aa62080a6b88e1afc"]},"id":"DLl_mpm9yXf3","outputId":"d07ce803-d465-4171-c3c2-3c34e98d31f4","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:46:55.107503Z","iopub.execute_input":"2025-04-08T05:46:55.107790Z","iopub.status.idle":"2025-04-08T05:46:56.853641Z","shell.execute_reply.started":"2025-04-08T05:46:55.107768Z","shell.execute_reply":"2025-04-08T05:46:56.852347Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"Map:   0%|          | 0/28599 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f5ded30bdf1c4faa8025ac083a12fbc8"}},"metadata":{}}],"execution_count":21},{"cell_type":"code","source":"# See what is it like again\n# 看看格式化后的数据是否正确\ndataset_format[495]","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2HxqPsCEzGF4","outputId":"38fa13f0-58af-4e76-9364-8842f1502a90","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:46:59.778715Z","iopub.execute_input":"2025-04-08T05:46:59.778999Z","iopub.status.idle":"2025-04-08T05:46:59.784786Z","shell.execute_reply.started":"2025-04-08T05:46:59.778976Z","shell.execute_reply":"2025-04-08T05:46:59.783887Z"}},"outputs":[{"execution_count":22,"output_type":"execute_result","data":{"text/plain":"{'Unnamed: 0': '173_0',\n 'system': '你是一个游戏剧情规划师。',\n 'user': '请根据游戏名编写游戏介绍：【游戏名】：Marguerite Sphere -マーガレット スフィア-。',\n 'assistant': '真要用最简洁语言来形容这城镇的话，那就是『花之街』。\\n这是充满异国情调的港口城市——白美根。\\n在这所到之处皆是野生雏菊的城镇里，有着一个传说。\\n——在拥有纯粹专一思念的少年少女面前，花之精灵将为他们送上，能够实现愿望的雏菊之花——\\n如此的白美根街住着一对双胞胎兄妹——此花大地、此花茉莉，如今他们都在私立白美根学园上学。\\n妹妹成绩优秀、头脑清晰，还有着能让学校男生们都为之着迷的可爱容貌。\\n可惜的是，与那容貌相对的，她却是不管对方是谁都尖酸刻薄相待的冰山利刃。\\n也正是这巨大反差，让她成为了备受他人注目的存在。\\n然而，其实她只是异常腼腆，以至于没能好好表现自己，着实楚楚可怜的妹妹罢了。\\n这样的她和哥哥大地，就在朋友们的簇拥下过着轻松明快的每一天。\\n那样的日子，想必会一直持续下去。没错，一直持续，直到今天……\\n「你还真是百年难得一遇的女难相呀……」\\n这些话出自于一位无意间碰到却被硬拉着占卜的占卜师之口。\\n在这点缀着纯白花朵的城镇里，如梦似幻又稍稍不可思议的恋爱故事就此拉开序幕……',\n 'formatted': '你是一个文字冒险游戏剧情策划师。根据我提供的数据和游戏信息，发挥你的想象力，写出一段以对白和叙事为主的游戏介绍。\\n\\n### Instruction:\\n请根据游戏名和游戏特色编写游戏介绍：【游戏名】：牛顿与苹果树 - ニュートンと林檎の樹  Newrin，【游戏特色】：性转， 萝莉， 秋野花， 剧情， 小仓结衣， R18， 纯爱， 黄油。\\n\\n### Response:\\n主人公 朝永修二和青梅竹马的一二三 四五为了寻找行踪不明的修二祖父，一个不留神就呼的一下穿越了。\\n回过神来眼前的是仅仅一名的少女。没错，有什么好隐藏的！\\n她就是后世被人敬仰那个传说中的近代科学之父，艾萨克#牛顿！\\n没错，牛顿是金发贫乳美少女！！！！！！！！！！！！！！！\\n不过此时的修二和四五还不知道这一切，没错知道的只有我这个天之音而已。\\n于是愣头愣脑的修二，就这么在牛顿的面前，在金发贫乳美少女的表情，\\n哗啦的一下，把那颗传说中的苹果树给干翻了？！！\\n【刚才……一瞬间，仿佛突然想到什么很重要的事情，但是又好像哪里不对的样子】\\n什么！天之音我也不禁悍然！\\n因为两人时空穿越的瞬间，正好是传说中牛顿在苹果树下悟出万有引力的瞬间！\\n哎呀呀呀，因为没有天之音的剧透，修二那个蠢货莽撞的行动，使得世纪的大发现就这么刷，刷刷刷，如同蒲公英的种子一般随风消散了。\\n之后，更是不用说由于这个蝴蝶效应的关系，使得历史变得一塌糊涂的情况了。\\n喂，童真崽，你打算怎么收拾这个残局啊！\\n时值17世纪！舞台乃是英国名门，那个有着10座大桥被人广为称赞的天才学府，剑桥大学！\\n修二到底，能否完成历史的修正，平安的回到现代呢！\\n由Laplacian赠送给大家“巨大的欢笑与些许的泪水”，忘却烦恼开心本心，时空穿越科学爱情喜剧美少女游戏！\\n\\n<eos>### Instruction:\\n请根据游戏名和游戏特色编写游戏介绍：【游戏名】：葬花·暗黑桃花源，【游戏特色】：民俗。\\n\\n### Response:\\n迷途者于深山误入传说中的桃花源。\\n桃花源的村民过了两千年与世隔绝的生活，维持着古代的生活习惯，迷途者逐渐融入村子，与村民们建立了良好的关系。\\n然而，桃花源的秘密也开始浮现——村民在白天时纯朴友好，晚上时则狂暴放纵，白天时清纯可爱的神女大人，晚上时则妖艳妩媚，村民们没有后代，也不会衰老.....\\n一切谜团似乎都与秦始皇渴求的“长生不老之术”有关......\\n终有一天，迷途者会看清这里的真相，然后做出选择：\\n——留下\\n还是离开？\\n\\n<eos>### Instruction:\\n请根据游戏名和游戏特色编写游戏介绍：【游戏名】：青春x好奇相伴的三角恋爱 - スキとスキとでサンカク恋愛，【游戏特色】：ASaProject， 卯衣， 颜艺社， 纯爱， 幼なじみ， 北見六花， 实妹， 幼驯染， 日常， 义妹， 搞笑， 废萌。\\n\\n### Response:\\n在某个乡下城镇过着学园生活的主人公「宗介」，因为父亲再婚的原因，从度过幼年时期后开始，就和义妹「七瑠」、青梅竹马「真帆」等人一起，在学园中的OTAKU部（通称B研）里享受着愉快的宅生活。\\n平稳的每一天……\\n然而，暑假还有近一个月就要开始的某一天，突然发生了事件。\\n「我一直都好想你……哥哥！！」\\n在幼年时期一直住在一起，之后被母亲带走的亲妹妹「铃」竟然回来了！\\n在学园、在家、在外面，静静地发挥着自己亲妹妹力量的亲妹妹。\\n一直显得不自在的义妹。\\n义妹：「就算你现在说妹妹什么的——这家伙（哥哥）早就已经被调教完成了——咕嘿嘿」\\n亲妹：「那我就再重新调教一次」\\n糟糕。我的两个妹妹糟透了。\\n本来已经是非常危急的状况了，却又突然出现了毫不讲理突然迷上自己的前辈「志衣菜」，\\n再加上青梅竹马「真帆」的情况也开始有点奇怪了。\\n这情况该怎么办，会变成怎么样呢！！\\n宗介就这样，被卷入了炎炎夏日都会冻上的三角关系\\n\\n<eos>'}"},"metadata":{}}],"execution_count":22},{"cell_type":"code","source":"# Shuffle the dataset for randomness\n# 混淆重排序，为了防止过拟合\nshuffled_dataset = dataset_format.shuffle(seed = 253831246211)\nshuffled_dataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:47:10.002134Z","iopub.execute_input":"2025-04-08T05:47:10.002473Z","iopub.status.idle":"2025-04-08T05:47:10.022374Z","shell.execute_reply.started":"2025-04-08T05:47:10.002444Z","shell.execute_reply":"2025-04-08T05:47:10.021609Z"}},"outputs":[{"execution_count":23,"output_type":"execute_result","data":{"text/plain":"Dataset({\n    features: ['Unnamed: 0', 'system', 'user', 'assistant', 'formatted'],\n    num_rows: 28599\n})"},"metadata":{}}],"execution_count":23},{"cell_type":"code","source":"# Slice the dataset to avoid entire loading that costs VRAM too much\n# 对数据集进行切片，以避免整个加载耗费过多的 VRAM，如果显存爆了调用这个\nif True:\n    #                                                   L   R + 1   step\n    sliced_dataset = shuffled_dataset.select(range(    0,   4200,    1))\n    print(sliced_dataset)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:47:39.797433Z","iopub.execute_input":"2025-04-08T05:47:39.797734Z","iopub.status.idle":"2025-04-08T05:47:39.805711Z","shell.execute_reply.started":"2025-04-08T05:47:39.797713Z","shell.execute_reply":"2025-04-08T05:47:39.804914Z"}},"outputs":[{"name":"stdout","text":"Dataset({\n    features: ['Unnamed: 0', 'system', 'user', 'assistant', 'formatted'],\n    num_rows: 4200\n})\n","output_type":"stream"}],"execution_count":24},{"cell_type":"markdown","source":"## 4. Setup the trainer","metadata":{"id":"dJzvujwnR_r2"}},{"cell_type":"code","source":"# Configurate the model to train\n# 设置你训练模型的训练参数\ntrainer = SFTTrainer(\n    model = _global_model,\n    tokenizer = _global_tokenizer,  \n    train_dataset = sliced_dataset,        # 这是你formatted格式化后，shuffle后的数据，也可以是slice后的数据\n    dataset_text_field = \"formatted\",      # 这个字段需要与之前format函数中新建的字段保持一致\n    max_seq_length = _global_model_max_seqlen,\n    dataset_num_proc = 4,                  # 建议设置为4如果你使用的是colab或者kaggle\n    packing = False,\n    args = TrainingArguments(\n        per_device_train_batch_size = 1,   # 每一个GPU训练的batchsize，建议设置为1或2或4或8如果你使用的是colab或者kaggle\n                                           # * 这是最推荐你们微调的参数，如果你的底模很大，选择更小的值；反之亦然\n                                           # * 如果提示爆显存了，那么设置成更小的值（最小为1）；但是，过小的值会使得训练变慢\n        gradient_accumulation_steps = 12,  # 梯度累积的步数，建议设置为4或8或12，8会使得梯度更稳定，4会防止局部鞍点，不能过高，否则梯度调整次数不够\n        warmup_steps = 5,                  # · 冷启动参数，在训练开始时，学习率通常从较小的值逐渐增加到设定的初始学习率，\n                                           # · 以避免训练初期模型不稳定或收敛困难的情况，定义了这一预热阶段所持续的训练步数\n                                           # · 在初次训练时候应该设置更大的值，例如10或者20；微调和instruct中可以设置很小的值\n        num_train_epochs = 1,              # 继续训练几个epoch，一般1足够了，如果你的数据集特别特殊且样本量较小，设为2\n                                           # · 什么是epoch和step? 一个step指的是一次数据传递，一个batch包含传递了所有数据的k个steps\n        learning_rate = 15e-5,             # 学习率，权重更新的速度，在微调中设置较低的值防止过拟合，不能过低，否则会陷入局部鞍点\n                                           # * 这是第二推荐你们微调的参数，如果你的数据是正常数据，约有大几千或者几万个，1e-4是推荐的值\n                                           # * 如果你的数据样本量很小，或者格式/语言并非常规的，可以考虑使用更大的值，但是封顶建议2e-4\n                                           # * 此外，如果你的batch_size设置的较大，等效learning_rate会降低，因此也建议选更大的值\n        fp16 = not is_bfloat16_supported(),\n        bf16 = is_bfloat16_supported(),\n        logging_steps = 1,\n        optim = \"adamw_8bit\",\n        weight_decay = 0.01,               # L2正则化，UP试了很久，不建议设置太大，否则会丢模型原始数据\n        lr_scheduler_type = \"linear\",\n        seed = _train_seed,\n        output_dir = \"outputs\",\n        report_to = \"none\",\n        dataloader_num_workers = 4,      \n        # If using multi-GPUS set these, 如果使用了多GPU，保留下面的内容否则注释掉\n        dataloader_pin_memory = True,\n        local_rank = -1, \n        ddp_find_unused_parameters = False,\n    ),\n)","metadata":{"id":"Vmj5pSNdSDau","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:48:53.892464Z","iopub.execute_input":"2025-04-08T05:48:53.892818Z","iopub.status.idle":"2025-04-08T05:49:04.764285Z","shell.execute_reply.started":"2025-04-08T05:48:53.892795Z","shell.execute_reply":"2025-04-08T05:49:04.763343Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"Unsloth: Tokenizing [\"formatted\"] (num_proc=4):   0%|          | 0/4200 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2c40e51aa23b4b0e9b572c3734537310"}},"metadata":{}}],"execution_count":25},{"cell_type":"code","source":"# GPU Stats, code is provided by unsloth\n# GPU数据，代码由unsloth提供\ngpu_stats = torch.cuda.get_device_properties(0)\nstart_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\nmax_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)\nprint(f\"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.\")\nprint(f\"{start_gpu_memory} GB of memory reserved.\")","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"CHghfVPaSeEe","outputId":"84074969-e8b4-4344-9952-02a8069d459f","trusted":true,"execution":{"iopub.status.busy":"2025-04-07T18:42:38.329074Z","iopub.execute_input":"2025-04-07T18:42:38.329424Z","iopub.status.idle":"2025-04-07T18:42:38.335980Z","shell.execute_reply.started":"2025-04-07T18:42:38.329396Z","shell.execute_reply":"2025-04-07T18:42:38.335010Z"}},"outputs":[{"name":"stdout","text":"GPU = Tesla T4. Max memory = 14.741 GB.\n7.486 GB of memory reserved.\n","output_type":"stream"}],"execution_count":45},{"cell_type":"markdown","source":"## 5. Officially train the model","metadata":{"id":"HMEAZdPJUGau"}},{"cell_type":"code","source":"# Call gc to clear cuda cache and release unused VRAM\n# 调用自动垃圾回收机制来强制清理未使用的显存对象，减少爆显存风险\n\nimport torch\nimport gc\n\ngc.collect()\ntorch.cuda.empty_cache()\ntorch.cuda.ipc_collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:49:21.828697Z","iopub.execute_input":"2025-04-08T05:49:21.828989Z","iopub.status.idle":"2025-04-08T05:49:22.327030Z","shell.execute_reply.started":"2025-04-08T05:49:21.828966Z","shell.execute_reply":"2025-04-08T05:49:22.326226Z"}},"outputs":[],"execution_count":27},{"cell_type":"code","source":"## Officially train the model\n## 正式开始训练你的模型，耗费时间（几十分钟到数小时）\ntrainer_stats = trainer.train()\nprint(\"Completed.\")\n\n# 什么时候停止？如果你的学习率太大，几十步一百步loss损失函数就到0.8/0.6这个位置了，那么\n# 哪怕训练数据没用完，停！ 因为接下来的都是过拟合(最简单的理解就是只会照猫画虎，不会举一反三)，训练loss不是越低越好。\n# 此时，自己测试一下，如果满意，导出保存。\n# 如果不满意，调低一点learning_rate，再从头训练。\n# 此外，如果你们的数据集中有大量数据重复或者高度相似，也有可能很快过拟合，请考虑数据集的问题。\n#\n# 你们一般不会见到欠拟合然后出大问题的情况。欠拟合就再训练：如果真遇到了，再跑一次这一行代码就可以。\n# 个人建议最终停止的位置是loss函数稳定到0.8 ~ 1.4之间的某一个阶段，例如已经稳定了30steps。\n# 如果你们愿意略微调整一下模型而严格防止过拟合，那么可以再1.2 ~ 1.6左右停止训练，或者增加weight_decay。","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":789},"id":"zkOjD1aYTfDt","outputId":"85a371d7-9d15-41d4-fa56-13f46fd04f42","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Show final training stats，code is provided by unsloth\n# 训练的统计数据，代码由unsloth提供\nused_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\nused_memory_for_lora = round(used_memory - start_gpu_memory, 3)\nused_percentage = round(used_memory         /max_memory*100, 3)\nlora_percentage = round(used_memory_for_lora/max_memory*100, 3)\nprint(f\"{trainer_stats.metrics['train_runtime']} seconds used for training.\")\nprint(f\"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training.\")\nprint(f\"Peak reserved memory = {used_memory} GB.\")\nprint(f\"Peak reserved memory for training = {used_memory_for_lora} GB.\")\nprint(f\"Peak reserved memory % of max memory = {used_percentage} %.\")\nprint(f\"Peak reserved memory for training % of max memory = {lora_percentage} %.\")","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"7QH2H_mCTkpu","outputId":"4fe81129-f0f6-45b1-870b-085493bfae30","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Inference Test","metadata":{"id":"3h_v1V-s1zuf"}},{"cell_type":"code","source":"# Reload the model, if you enter for the second time\n# 重新加载模型，如果你保存了然后第二次进来\nif False:\n    from unsloth import FastLanguageModel\n\n    _global_model, _global_tokenizer = FastLanguageModel.from_pretrained(\n        model_name=\"my_gameintro_gemma9b\",\n        max_seq_length=_global_model_max_seqlen,\n        dtype = _global_model_dtype,\n        load_in_4bit = _global_model_load_in_4bit\n    )\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# If you plan to infer rather than continue to train, call this\n# 如果你希望推理而不是继续训练，调用如下这行代码，训练一定不要调\nif False:\n    FastLanguageModel.for_inference(_global_model)  # Enable native 2x faster inference","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inference\n# 推理测试\nFastLanguageModel.for_inference(_global_model)\ninputs = _global_tokenizer(\n[\n    # Use Infer when doing inference\n    alpaca_prompt_infer.format(\n        \"你是一个游戏剧情规划师。请你根据我提供的游戏名和游戏特色规划剧情，写出一段引人入胜的游戏介绍。\", # system\n        \"请根据游戏名编写游戏介绍：【游戏名】：在终焉的世界里寻找盛开的花。\", # input\n        \"\", # output - 留空等待AI生成\n    )\n], return_tensors = \"pt\").to(\"cuda\")\n\noutputs = _global_model.generate(**inputs, max_new_tokens = 256, use_cache = 2048)\n_global_tokenizer.batch_decode(outputs)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9ozwL6jWTtrG","outputId":"5ff93dec-d3a1-4f46-e4e6-64df5b9ecbf9","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T01:13:17.363610Z","iopub.execute_input":"2025-04-08T01:13:17.363960Z","iopub.status.idle":"2025-04-08T01:13:50.074870Z","shell.execute_reply.started":"2025-04-08T01:13:17.363934Z","shell.execute_reply":"2025-04-08T01:13:50.074074Z"}},"outputs":[{"execution_count":52,"output_type":"execute_result","data":{"text/plain":"['<bos>你是一个游戏剧情规划师。请你根据我提供的游戏名和游戏特色规划剧情，写出一段引人入胜的游戏介绍。\\n\\n### Instruction:\\n请根据游戏名编写游戏介绍：【游戏名】：在终焉的世界里寻找盛开的花。\\n\\n### Response:\\n在某个世界中，存在着一位女神。\\n女神创造了万物，创造了世界。\\n她创造了无数的生物，创造了广阔的土地，创造了蔚蓝的大海。\\n她创造了人类，创造了动物，创造了植物。\\n她创造了天空，创造了大地，创造了时间。\\n她创造了世界。\\n女神创造了世界，她创造了万物。\\n然而，女神却无法创造出令自己满意的“智慧生命体”。\\n于是，女神反复尝试着创造出智慧生命体。\\n然而，创造出来的智慧生命体却无法成长，都是些只会本能地行动的傻瓜。\\n女神对这些傻瓜们感到厌恶，她将他们全部贬到了广袤无边的“荒野”之地。\\n在那里，智慧生命体无法生存。\\n在那里，智慧生命体只能被当作牲畜对待。\\n在那里，智慧生命体只能被当作愚蠢的动物对待。\\n女神将这些傻瓜们贬到了荒野之地。\\n然而，傻瓜们最终在荒野之地创造出了文明。\\n他们创造出了国家，他们创造出了秩序。\\n他们创造出了智慧。\\n被贬到了荒野']"},"metadata":{}}],"execution_count":52},{"cell_type":"code","source":"# Inference in stream mode\n# 流式推理测试\nFastLanguageModel.for_inference(_global_model)\ninputs = _global_tokenizer(\n[\n    # Use Infer when doing inference\n    alpaca_prompt_infer.format(\n        \"你是一个游戏剧情规划师。请你根据我提供的游戏名和游戏特色规划剧情，写出一段引人入胜的游戏介绍。\", # system\n        \"请根据游戏名和游戏特色编写游戏介绍：【游戏名】：风陇之歌 ~Tracking the footprints of time~，【游戏特色】：奇幻， 哲学， 冒险， 宗教， 神话， 白毛。\", # input\n        \"\", # output - 留空等待AI生成\n    )\n], return_tensors = \"pt\").to(\"cuda\")\n\nfrom transformers import TextStreamer\ntext_streamer = TextStreamer(_global_tokenizer)\n_ = _global_model.generate(**inputs, streamer = text_streamer, max_new_tokens = 2048)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"fIJ415USTvOe","outputId":"95b525e2-f2b3-4707-f023-85d57790e502","trusted":true,"execution":{"iopub.status.busy":"2025-04-08T05:51:26.714888Z","iopub.execute_input":"2025-04-08T05:51:26.715236Z","iopub.status.idle":"2025-04-08T05:51:40.429592Z","shell.execute_reply.started":"2025-04-08T05:51:26.715210Z","shell.execute_reply":"2025-04-08T05:51:40.428688Z"}},"outputs":[{"name":"stdout","text":"<bos>你是一个游戏剧情规划师。请你根据我提供的游戏名和游戏特色规划剧情，写出一段引人入胜的游戏介绍。\n\n### Instruction:\n请根据游戏名和游戏特色编写游戏介绍：【游戏名】：风陇之歌 ~Tracking the footprints of time~，【游戏特色】：奇幻， 哲学， 冒险， 宗教， 神话， 白毛。\n\n### Response:\n《风岭之歌》是一款以“时间”为主题的纯文字冒险游戏。\n在故事中，玩家将扮演一名普通的大学生，因意外事故而穿越时空，\n与神秘美少女“风岭”一同，在两个时代中展开寻人寻物等一系列任务。\n随着故事的推进，真相也逐渐浮出水面……\n\n<eos>\n","output_type":"stream"}],"execution_count":29},{"cell_type":"markdown","source":"## 7. Save the model","metadata":{"id":"ttGAEGntVavo"}},{"cell_type":"code","source":"# Save model as the native huggingface version - local\n# 保存模型为原生的huggingface模型到本地\n_global_model.save_pretrained(\"my_gameintro_gemma9b\")\n_global_tokenizer.save_pretrained(\"my_gameintro_gemma9b\")\n\n# Save the merged - locel\n# 保存融合的模型(包括底模)到本地\n_global_model.save_pretrained_merged(\"my_gameintro_gemma9b_merged\", _global_tokenizer, save_method = \"merged_16bit\",)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"_boIgrvbT30O","outputId":"3e0bd437-23c0-4bb9-9ecb-f7c0948b5ad3","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save model as the native huggingface version - hf, costs time\n# 保存模型为原生的huggingface模型到hf模型站，消耗时间！请为你的运行时留足时间\n_global_model.push_to_hub(\"DOFOFFICIAL/ThisIsNewTmp\", token = \"hf_...\")\n_global_tokenizer.push_to_hub(\"DOFOFFICIAL/ThisIsNewTmp\", token = \"hf_...\") \n\n# Save the merged - hf, costs time\n# 保存融合的模型(包括底模)到hf模型站，消耗时间！请为你的运行时留足时间\n_global_model.push_to_hub_merged(\"DOFOFFICIAL/NathUI-Tutorial\", _global_tokenizer, save_method = \"merged_16bit\", token = \"hf_...\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save as quanted, costs time\n# 保存为量化后的模型，消耗时间！请为你的运行时留足时间\n\n# Save to Q8_0\n# 保存为量化的gguf Q8_0\n# _global_model.save_pretrained_gguf(\"my_gameintro_gemma9b_Q8_0\", _global_tokenizer, quantization_method = \"q8_0\")\n_global_model.push_to_hub_gguf(\"DOFOFFICIAL/ThisIsTmp\", _global_tokenizer, quantization_method = \"q8_0\", token = \"hf_...\")\n\n# Save to Q4_K_M\n# 保存为量化的gguf Q4_K_M\n# _global_model.save_pretrained_gguf(\"my_gameintro_gemma9b_Q4_K_M\", _global_tokenizer, quantization_method = \"q4_K_M\")\n_global_model.push_to_hub_gguf(\"DOFOFFICIAL/ThisIsTmp\", _global_tokenizer, quantization_method = \"q4_K_M\", token = \"hf_...\")\n","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"tHIKsMlqVwB_","outputId":"c0631069-7724-4f1d-d7cc-24d12786192a","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Modified Auther NathMath, open-sourced with Apache-2.0 Licence\n# 修改作者：NathMath，以Apache-2.0 Licence许可证开源","metadata":{"id":"3AmGcJfh3B_o","trusted":true},"outputs":[],"execution_count":null}]}