codeShare commited on
Commit
994f0bf
·
verified ·
1 Parent(s): ef280ec

Upload cleanup gemma4 training set.ipynb

Browse files
gemma4_training/cleanup gemma4 training set.ipynb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyPWE/shF+6tOopgDZikYCIQ"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"version_major":2,"version_minor":0,"state":{"fa1a436b23ba44b8b3470ab49f067d31":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_cc466e8b65e64b41b0425cd1f1197c9c","IPY_MODEL_75270d3bec354b508d6bafad8b9bce2e","IPY_MODEL_e49bc06cfe734df9adc2cf04df196a8a"],"layout":"IPY_MODEL_9cfd5a292d024262b0d5fa0309a23f0c"}},"cc466e8b65e64b41b0425cd1f1197c9c":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a5a9debb37c54cabb000a16f2c4cf026","placeholder":"​","style":"IPY_MODEL_24ee773c35ad43a7993252a91cb4db1c","value":"gemma_train_set.7z: reconstructing file: 100%"}},"75270d3bec354b508d6bafad8b9bce2e":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_d765e84bccd146079707e484b6080cb1","max":1639741856,"min":0,"orientation":"horizontal","style":"IPY_MODEL_26a89f6547fc4450835cba30066f4665","value":1639741856}},"e49bc06cfe734df9adc2cf04df196a8a":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a3a9165af81b423983933dfbcd8a602e","placeholder":"​","style":"IPY_MODEL_4fc4d3141f7e4f20ab7a2317c49c98e1","value":" 1.64GB / 1.64GB, 88.7MB/s  "}},"9cfd5a292d024262b0d5fa0309a23f0c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a5a9debb37c54cabb000a16f2c4cf026":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"24ee773c35ad43a7993252a91cb4db1c":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"d765e84bccd146079707e484b6080cb1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"26a89f6547fc4450835cba30066f4665":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"a3a9165af81b423983933dfbcd8a602e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4fc4d3141f7e4f20ab7a2317c49c98e1":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a3718b576c674f14a9b5d40ee83c8041":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_668624ba30404659bfbcdf23e2f4bd72","IPY_MODEL_3c7abadad7a44326b2d3ac23e94cd9bc","IPY_MODEL_23b1b2f35fe54fbe85066d52a1f404a8"],"layout":"IPY_MODEL_efcf0abd3106425a811826bd5fda0a60"}},"668624ba30404659bfbcdf23e2f4bd72":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d5e3f7488d9c4d298fb7135971c2244f","placeholder":"​","style":"IPY_MODEL_fcc9823d544c475fb0b273ee2caab999","value":"gemma_train_set.7z: downloading bytes: "}},"3c7abadad7a44326b2d3ac23e94cd9bc":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_811c091b54b24581859c4906beb4b65a","max":1639741856,"min":0,"orientation":"horizontal","style":"IPY_MODEL_8dc33d5f99be49589988b7046bd393ec","value":1639741856}},"23b1b2f35fe54fbe85066d52a1f404a8":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_f0310d81ff4048bbbfa1f893ebfc1cea","placeholder":"​","style":"IPY_MODEL_67203f3cb20c4cb6aea82146a857809e","value":" 1.64GB,  119MB/s  "}},"efcf0abd3106425a811826bd5fda0a60":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d5e3f7488d9c4d298fb7135971c2244f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"fcc9823d544c475fb0b273ee2caab999":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"811c091b54b24581859c4906beb4b65a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8dc33d5f99be49589988b7046bd393ec":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"f0310d81ff4048bbbfa1f893ebfc1cea":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"67203f3cb20c4cb6aea82146a857809e":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"573a4fa6060c4c1eb3e988176f0bfe0f":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_ec4499e447c349ae8d1f07a0e1d4121a","IPY_MODEL_dd69c15f4e104b5ba7dd5d92a9686a2f","IPY_MODEL_8025f80888a948cfb5b3f2cdcfabf61b"],"layout":"IPY_MODEL_8b478cc424304e76ade032247a4c57e9"}},"ec4499e447c349ae8d1f07a0e1d4121a":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b8191cd05f904fc38a2dcfb5d8d99c63","placeholder":"​","style":"IPY_MODEL_1f199077c71a4201b27ac1f3030cb165","value":"Cleaning training dataset:  24%"}},"dd69c15f4e104b5ba7dd5d92a9686a2f":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_4d15bf1d38694ddba7e8f6deea329057","max":6843,"min":0,"orientation":"horizontal","style":"IPY_MODEL_de867f0c49364d679854b9522babe58b","value":1610}},"8025f80888a948cfb5b3f2cdcfabf61b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_57eed4320f0b4333af2eb5e6c5d2debd","placeholder":"​","style":"IPY_MODEL_0b7b52ca681c4ff0a23c9b4310f6c112","value":" 1610/6843 [03:19&lt;09:34,  9.10it/s]"}},"8b478cc424304e76ade032247a4c57e9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b8191cd05f904fc38a2dcfb5d8d99c63":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1f199077c71a4201b27ac1f3030cb165":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"4d15bf1d38694ddba7e8f6deea329057":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"de867f0c49364d679854b9522babe58b":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"57eed4320f0b4333af2eb5e6c5d2debd":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0b7b52ca681c4ff0a23c9b4310f6c112":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}}},"cells":[{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":385,"referenced_widgets":["fa1a436b23ba44b8b3470ab49f067d31","cc466e8b65e64b41b0425cd1f1197c9c","75270d3bec354b508d6bafad8b9bce2e","e49bc06cfe734df9adc2cf04df196a8a","9cfd5a292d024262b0d5fa0309a23f0c","a5a9debb37c54cabb000a16f2c4cf026","24ee773c35ad43a7993252a91cb4db1c","d765e84bccd146079707e484b6080cb1","26a89f6547fc4450835cba30066f4665","a3a9165af81b423983933dfbcd8a602e","4fc4d3141f7e4f20ab7a2317c49c98e1","a3718b576c674f14a9b5d40ee83c8041","668624ba30404659bfbcdf23e2f4bd72","3c7abadad7a44326b2d3ac23e94cd9bc","23b1b2f35fe54fbe85066d52a1f404a8","efcf0abd3106425a811826bd5fda0a60","d5e3f7488d9c4d298fb7135971c2244f","fcc9823d544c475fb0b273ee2caab999","811c091b54b24581859c4906beb4b65a","8dc33d5f99be49589988b7046bd393ec","f0310d81ff4048bbbfa1f893ebfc1cea","67203f3cb20c4cb6aea82146a857809e","573a4fa6060c4c1eb3e988176f0bfe0f","ec4499e447c349ae8d1f07a0e1d4121a","dd69c15f4e104b5ba7dd5d92a9686a2f","8025f80888a948cfb5b3f2cdcfabf61b","8b478cc424304e76ade032247a4c57e9","b8191cd05f904fc38a2dcfb5d8d99c63","1f199077c71a4201b27ac1f3030cb165","4d15bf1d38694ddba7e8f6deea329057","de867f0c49364d679854b9522babe58b","57eed4320f0b4333af2eb5e6c5d2debd","0b7b52ca681c4ff0a23c9b4310f6c112"]},"id":"jKAJshGiWmQ7","outputId":"a787aed7-4733-472e-f810-38d00dbbd4cc"},"outputs":[{"output_type":"stream","name":"stdout","text":["======================================================================\n","KROMOSS TRAINING DATA CLEANING\n","======================================================================\n","\n","Downloading original dataset...\n"]},{"output_type":"display_data","data":{"text/plain":["gemma_train_set.7z: reconstructing file: 0%| | 0.00B / 1.64GB "],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fa1a436b23ba44b8b3470ab49f067d31"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["gemma_train_set.7z: downloading bytes: | 0.00B "],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a3718b576c674f14a9b5d40ee83c8041"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Download completed.\n","Archive size: 1.527 GB\n","\n","Extracting dataset...\n","Extraction completed.\n","\n","Training items found: 6843\n","\n","Filtering and cleaning training items...\n"]},{"output_type":"display_data","data":{"text/plain":["Cleaning training dataset: 0%| | 0/6843 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"573a4fa6060c4c1eb3e988176f0bfe0f"}},"metadata":{}}],"source":["\n","# ============================================================\n","# KROMOSS TRAINING DATASET\n","# FILTER + CLEAN + RE-ENCRYPT + VALIDATE + COMMIT\n","# ============================================================\n","\n","# ============================================================\n","# 1. INSTALL DEPENDENCIES\n","# ============================================================\n","\n","!pip install -q huggingface_hub py7zr cryptography tqdm pillow\n","\n","\n","# ============================================================\n","# 2. IMPORTS\n","# ============================================================\n","\n","import os\n","import re\n","import gc\n","import shutil\n","import hashlib\n","import secrets\n","import tempfile\n","\n","from pathlib import Path\n","from collections import defaultdict\n","\n","from tqdm.auto import tqdm\n","from google.colab import userdata\n","\n","from huggingface_hub import (\n"," login,\n"," hf_hub_download,\n"," HfApi\n",")\n","\n","from cryptography.hazmat.primitives.ciphers.aead import AESGCM\n","from cryptography.hazmat.primitives.kdf.scrypt import Scrypt\n","\n","\n","# ============================================================\n","# 3. CONFIGURATION\n","# ============================================================\n","\n","HF_TOKEN = userdata.get(\"HF_TOKEN\")\n","\n","if not HF_TOKEN:\n"," raise ValueError(\"HF_TOKEN missing from Colab Secrets.\")\n","\n","# Store your actual existing encryption password in\n","# Colab Secrets under KROMOSS_PASSWORD.\n","PASSWORD = 'blueberry' #userdata.get(\"KROMOSS_PASSWORD\")\n","\n","if not PASSWORD:\n"," raise ValueError(\n"," \"KROMOSS_PASSWORD missing from Colab Secrets.\"\n"," )\n","\n","login(token=HF_TOKEN)\n","\n","api = HfApi(token=HF_TOKEN)\n","\n","REPO_ID = \"codeShare/kromoss-lora-exports\"\n","REPO_TYPE = \"model\"\n","\n","ARCHIVE_NAME = \"gemma_train_set.7z\"\n","\n","PCENC_MAGIC = b\"PCENC1\"\n","\n","MINIMUM_CHARACTERS = 50\n","\n","# Remove whole words only, ignoring capitalization.\n","EXCLUDED_WORDS = re.compile(\n"," r\"\\b(?:remove|improve)\\b\",\n"," re.IGNORECASE\n",")\n","\n","WORK_DIR = Path(\"/content/kromoss_cleaning\")\n","\n","SOURCE_ARCHIVE = WORK_DIR / \"original.7z\"\n","\n","EXTRACT_DIR = WORK_DIR / \"dataset\"\n","\n","OUTPUT_ARCHIVE = WORK_DIR / \"gemma_train_set_cleaned.7z\"\n","\n","WORK_DIR.mkdir(parents=True, exist_ok=True)\n","\n","print(\"=\" * 70)\n","print(\"KROMOSS TRAINING DATA CLEANING\")\n","print(\"=\" * 70)\n","\n","\n","# ============================================================\n","# 4. ENCRYPTION AND DECRYPTION\n","# Compatible with the original PCENC1 implementation.\n","# ============================================================\n","\n","def derive_key(password, salt):\n","\n"," return Scrypt(\n"," salt=salt,\n"," length=32,\n"," n=2**14,\n"," r=8,\n"," p=1\n"," ).derive(password.encode(\"utf-8\"))\n","\n","\n","def decrypt_file_bytes(data):\n","\n"," if not data.startswith(PCENC_MAGIC):\n"," return data\n","\n"," if len(data) < 50:\n"," raise ValueError(\"Invalid or truncated PCENC1 file.\")\n","\n"," salt = data[6:22]\n"," nonce = data[22:34]\n"," ciphertext = data[34:]\n","\n"," key = derive_key(PASSWORD, salt)\n","\n"," return AESGCM(key).decrypt(\n"," nonce,\n"," ciphertext,\n"," PCENC_MAGIC\n"," )\n","\n","\n","def encrypt_file_bytes(data):\n","\n"," if data.startswith(PCENC_MAGIC):\n"," data = decrypt_file_bytes(data)\n","\n"," salt = secrets.token_bytes(16)\n"," nonce = secrets.token_bytes(12)\n","\n"," key = derive_key(PASSWORD, salt)\n","\n"," ciphertext = AESGCM(key).encrypt(\n"," nonce,\n"," data,\n"," PCENC_MAGIC\n"," )\n","\n"," return (\n"," PCENC_MAGIC\n"," + salt\n"," + nonce\n"," + ciphertext\n"," )\n","\n","\n","def read_plaintext(path):\n","\n"," return decrypt_file_bytes(path.read_bytes())\n","\n","\n","def write_encrypted(path, plaintext):\n","\n"," path.write_bytes(\n"," encrypt_file_bytes(plaintext)\n"," )\n","\n","\n","def fingerprint(data):\n","\n"," return hashlib.sha256(data).digest()\n","\n","\n","# ============================================================\n","# 5. DOWNLOAD ORIGINAL DATASET\n","# ============================================================\n","\n","print(\"\\nDownloading original dataset...\")\n","\n","downloaded = hf_hub_download(\n"," repo_id=REPO_ID,\n"," filename=ARCHIVE_NAME,\n"," repo_type=REPO_TYPE,\n"," token=HF_TOKEN,\n"," force_download=True\n",")\n","\n","shutil.copy2(\n"," downloaded,\n"," SOURCE_ARCHIVE\n",")\n","\n","print(\"Download completed.\")\n","print(\"Archive size:\", round(\n"," SOURCE_ARCHIVE.stat().st_size / (1024**3), 3\n","), \"GB\")\n","\n","\n","# ============================================================\n","# 6. EXTRACT DATASET\n","# ============================================================\n","\n","import py7zr\n","\n","if EXTRACT_DIR.exists():\n"," shutil.rmtree(EXTRACT_DIR)\n","\n","EXTRACT_DIR.mkdir(parents=True, exist_ok=True)\n","\n","print(\"\\nExtracting dataset...\")\n","\n","with py7zr.SevenZipFile(\n"," SOURCE_ARCHIVE,\n"," mode=\"r\",\n"," password=PASSWORD\n",") as archive:\n","\n"," archive.extractall(path=EXTRACT_DIR)\n","\n","print(\"Extraction completed.\")\n","\n","\n","# ============================================================\n","# 7. IDENTIFY TRAINING ITEMS\n","# Each item consists of:\n","# ID.jpg\n","# ID_input.txt\n","# ID_output.txt\n","# ============================================================\n","\n","image_files = sorted(\n"," [\n"," p for p in EXTRACT_DIR.glob(\"*.jpg\")\n"," if p.stem.isdigit()\n"," ],\n"," key=lambda p: int(p.stem)\n",")\n","\n","if not image_files:\n"," raise RuntimeError(\n"," \"No numeric JPG training items found.\"\n"," )\n","\n","print(\"\\nTraining items found:\", len(image_files))\n","\n","\n","# ============================================================\n","# 8. TEXT CLEANING FUNCTION\n","# ============================================================\n","\n","def clean_text(text):\n","\n"," if not isinstance(text, str):\n"," return None\n","\n"," # Normalize whitespace at the beginning and end.\n"," text = text.strip()\n","\n"," # Remove leading punctuation, symbols, digits and\n"," # whitespace while preserving the actual text.\n"," #\n"," # Example:\n"," # \"' , A red apple\" -> \"A red apple\"\n"," #\n"," # Unicode letters are supported.\n","\n"," match = re.search(r\"[^\\W\\d_]\", text, flags=re.UNICODE)\n","\n"," if match is None:\n"," return None\n","\n"," text = text[match.start():]\n","\n"," # Capitalize the first alphabetic character.\n"," text = text[0].upper() + text[1:]\n","\n"," return text\n","\n","\n","# ============================================================\n","# 9. FILTER AND CLEAN ALL TRAINING ITEMS\n","# ============================================================\n","\n","print(\"\\nFiltering and cleaning training items...\")\n","\n","stats = {\n"," \"original\": len(image_files),\n"," \"keyword_removed\": 0,\n"," \"short_removed\": 0,\n"," \"invalid_removed\": 0,\n"," \"duplicates_removed\": 0,\n"," \"retained\": 0,\n"," \"cleaned_texts\": 0\n","}\n","\n","# Deduplicate cleaned output descriptions.\n","seen_outputs = defaultdict(list)\n","\n","# Track removed IDs.\n","removed_ids = []\n","\n","for image_path in tqdm(\n"," image_files,\n"," desc=\"Cleaning training dataset\"\n","):\n","\n"," item_id = image_path.stem\n","\n"," input_path = EXTRACT_DIR / f\"{item_id}_input.txt\"\n"," output_path = EXTRACT_DIR / f\"{item_id}_output.txt\"\n","\n"," # --------------------------------------------------------\n"," # VERIFY COMPLETE TRAINING ITEM\n"," # --------------------------------------------------------\n","\n"," if not input_path.is_file() or not output_path.is_file():\n","\n"," raise RuntimeError(\n"," f\"Incomplete training item: {item_id}. \"\n"," \"Aborting without uploading.\"\n"," )\n","\n"," # --------------------------------------------------------\n"," # DECRYPT AND READ TEXT\n"," # --------------------------------------------------------\n","\n"," try:\n","\n"," input_text = read_plaintext(\n"," input_path\n"," ).decode(\"utf-8\")\n","\n"," output_text = read_plaintext(\n"," output_path\n"," ).decode(\"utf-8\")\n","\n"," except Exception as error:\n","\n"," raise RuntimeError(\n"," f\"Cannot decrypt or read item {item_id}: {error}\"\n"," ) from error\n","\n"," # --------------------------------------------------------\n"," # RULE 1: REMOVE EXCLUDED WORDS\n"," # Search both input and output.\n"," # --------------------------------------------------------\n","\n"," if (\n"," EXCLUDED_WORDS.search(input_text)\n"," or EXCLUDED_WORDS.search(output_text)\n"," ):\n","\n"," stats[\"keyword_removed\"] += 1\n","\n"," removed_ids.append(item_id)\n","\n"," image_path.unlink()\n"," input_path.unlink()\n"," output_path.unlink()\n","\n"," continue\n","\n"," # --------------------------------------------------------\n"," # RULE 2: CLEAN LEADING CHARACTERS\n"," # --------------------------------------------------------\n","\n"," cleaned_input = clean_text(input_text)\n"," cleaned_output = clean_text(output_text)\n","\n"," if cleaned_input is None or cleaned_output is None:\n","\n"," stats[\"invalid_removed\"] += 1\n","\n"," removed_ids.append(item_id)\n","\n"," image_path.unlink()\n"," input_path.unlink()\n"," output_path.unlink()\n","\n"," continue\n","\n"," # --------------------------------------------------------\n"," # RULE 3: REMOVE TEXT SHORTER THAN 50 CHARACTERS\n"," # Count after cleaning.\n"," # --------------------------------------------------------\n","\n"," if (\n"," len(cleaned_input) < MINIMUM_CHARACTERS\n"," or len(cleaned_output) < MINIMUM_CHARACTERS\n"," ):\n","\n"," stats[\"short_removed\"] += 1\n","\n"," removed_ids.append(item_id)\n","\n"," image_path.unlink()\n"," input_path.unlink()\n"," output_path.unlink()\n","\n"," continue\n","\n"," # --------------------------------------------------------\n"," # RULE 4: DEDUPLICATE CLEANED OUTPUTS\n"," # --------------------------------------------------------\n","\n"," output_bytes = cleaned_output.encode(\"utf-8\")\n","\n"," output_hash = fingerprint(output_bytes)\n","\n"," duplicate = any(\n"," previous == output_bytes\n"," for previous in seen_outputs[output_hash]\n"," )\n","\n"," if duplicate:\n","\n"," stats[\"duplicates_removed\"] += 1\n","\n"," removed_ids.append(item_id)\n","\n"," image_path.unlink()\n"," input_path.unlink()\n"," output_path.unlink()\n","\n"," continue\n","\n"," seen_outputs[output_hash].append(output_bytes)\n","\n"," # --------------------------------------------------------\n"," # SAVE CLEANED PLAINTEXT\n"," # --------------------------------------------------------\n","\n"," input_path.write_text(\n"," cleaned_input,\n"," encoding=\"utf-8\"\n"," )\n","\n"," output_path.write_text(\n"," cleaned_output,\n"," encoding=\"utf-8\"\n"," )\n","\n"," stats[\"cleaned_texts\"] += 2\n"," stats[\"retained\"] += 1\n","\n","\n","# ============================================================\n","# 10. VERIFY ALL RETAINED ITEMS\n","# ============================================================\n","\n","print(\"\\nValidating cleaned dataset...\")\n","\n","final_images = sorted(\n"," [\n"," p for p in EXTRACT_DIR.glob(\"*.jpg\")\n"," if p.stem.isdigit()\n"," ],\n"," key=lambda p: int(p.stem)\n",")\n","\n","if len(final_images) != stats[\"retained\"]:\n"," raise RuntimeError(\"Retained item count mismatch.\")\n","\n","validation_seen = {}\n","\n","for image_path in tqdm(\n"," final_images,\n"," desc=\"Validating cleaned text\"\n","):\n","\n"," item_id = image_path.stem\n","\n"," input_path = EXTRACT_DIR / f\"{item_id}_input.txt\"\n"," output_path = EXTRACT_DIR / f\"{item_id}_output.txt\"\n","\n"," if not input_path.exists() or not output_path.exists():\n"," raise RuntimeError(\n"," f\"Incomplete item after cleaning: {item_id}\"\n"," )\n","\n"," input_text = input_path.read_text(encoding=\"utf-8\")\n"," output_text = output_path.read_text(encoding=\"utf-8\")\n","\n"," # No forbidden words.\n"," if (\n"," EXCLUDED_WORDS.search(input_text)\n"," or EXCLUDED_WORDS.search(output_text)\n"," ):\n"," raise RuntimeError(\n"," f\"Forbidden word remains in item {item_id}\"\n"," )\n","\n"," # Minimum character length.\n"," if (\n"," len(input_text) < MINIMUM_CHARACTERS\n"," or len(output_text) < MINIMUM_CHARACTERS\n"," ):\n"," raise RuntimeError(\n"," f\"Short text remains in item {item_id}\"\n"," )\n","\n"," # Both must begin with a capitalized letter.\n"," if not input_text[0].isupper() or not output_text[0].isupper():\n"," raise RuntimeError(\n"," f\"Capitalization error in item {item_id}\"\n"," )\n","\n"," if not input_text[0].isalpha() or not output_text[0].isalpha():\n"," raise RuntimeError(\n"," f\"Invalid leading character in item {item_id}\"\n"," )\n","\n"," # Check duplicate output descriptions.\n"," output_bytes = output_text.encode(\"utf-8\")\n"," output_hash = fingerprint(output_bytes)\n","\n"," if output_hash in validation_seen:\n","\n"," if output_bytes in validation_seen[output_hash]:\n"," raise RuntimeError(\n"," f\"Duplicate output remains: {item_id}\"\n"," )\n","\n"," validation_seen.setdefault(\n"," output_hash, []\n"," ).append(output_bytes)\n","\n","print(\"All text validation checks passed.\")\n","\n","\n","# ============================================================\n","# 11. RE-ENCRYPT ALL RETAINED FILES\n","# ============================================================\n","\n","print(\"\\nRe-encrypting retained training items...\")\n","\n","encrypted_count = 0\n","\n","for image_path in tqdm(\n"," final_images,\n"," desc=\"Re-encrypting dataset\"\n","):\n","\n"," item_id = image_path.stem\n","\n"," trio = [\n"," image_path,\n"," EXTRACT_DIR / f\"{item_id}_input.txt\",\n"," EXTRACT_DIR / f\"{item_id}_output.txt\"\n"," ]\n","\n"," for path in trio:\n","\n"," plaintext = read_plaintext(path)\n","\n"," encrypted = encrypt_file_bytes(plaintext)\n","\n"," path.write_bytes(encrypted)\n","\n"," encrypted_count += 1\n","\n","\n","# ============================================================\n","# 12. VERIFY ENCRYPTION\n","# ============================================================\n","\n","print(\"\\nVerifying encryption...\")\n","\n","verified_count = 0\n","\n","for image_path in tqdm(\n"," final_images,\n"," desc=\"Verifying PCENC1\"\n","):\n","\n"," item_id = image_path.stem\n","\n"," trio = [\n"," image_path,\n"," EXTRACT_DIR / f\"{item_id}_input.txt\",\n"," EXTRACT_DIR / f\"{item_id}_output.txt\"\n"," ]\n","\n"," for path in trio:\n","\n"," encrypted_data = path.read_bytes()\n","\n"," if not encrypted_data.startswith(PCENC_MAGIC):\n","\n"," raise RuntimeError(\n"," f\"Missing PCENC1 header: {path.name}\"\n"," )\n","\n"," # Authenticate and decrypt to verify.\n"," decrypt_file_bytes(encrypted_data)\n","\n"," verified_count += 1\n","\n","if verified_count != encrypted_count:\n","\n"," raise RuntimeError(\n"," \"Encryption verification count mismatch.\"\n"," )\n","\n","if encrypted_count != len(final_images) * 3:\n","\n"," raise RuntimeError(\n"," \"Expected exactly three encrypted files per item.\"\n"," )\n","\n","print(\"Encryption successfully verified.\")\n","\n","\n","# ============================================================\n","# 13. CREATE NEW ENCRYPTED 7Z ARCHIVE\n","# ============================================================\n","\n","print(\"\\nCreating cleaned archive...\")\n","\n","if OUTPUT_ARCHIVE.exists():\n"," OUTPUT_ARCHIVE.unlink()\n","\n","archive_files = sorted(\n"," p for p in EXTRACT_DIR.rglob(\"*\")\n"," if p.is_file()\n",")\n","\n","with py7zr.SevenZipFile(\n"," OUTPUT_ARCHIVE,\n"," mode=\"w\",\n"," password=PASSWORD,\n"," header_encryption=True\n",") as archive:\n","\n"," for path in tqdm(\n"," archive_files,\n"," desc=\"Creating archive\"\n"," ):\n","\n"," relative_path = path.relative_to(\n"," EXTRACT_DIR\n"," ).as_posix()\n","\n"," archive.write(\n"," path,\n"," arcname=relative_path\n"," )\n","\n","if not OUTPUT_ARCHIVE.exists():\n"," raise RuntimeError(\"Archive creation failed.\")\n","\n","print(\"Archive created:\", OUTPUT_ARCHIVE)\n","\n","\n","# ============================================================\n","# 14. VERIFY ARCHIVE STRUCTURE\n","# ============================================================\n","\n","print(\"\\nVerifying archive...\")\n","\n","expected_files = {\n"," p.relative_to(EXTRACT_DIR).as_posix()\n"," for p in archive_files\n","}\n","\n","with py7zr.SevenZipFile(\n"," OUTPUT_ARCHIVE,\n"," mode=\"r\",\n"," password=PASSWORD\n",") as archive:\n","\n"," actual_files = set(archive.getnames())\n","\n","if expected_files != actual_files:\n","\n"," raise RuntimeError(\n"," \"Archive structure verification failed.\"\n"," )\n","\n","print(\"Archive structure verified.\")\n","\n","\n","# ============================================================\n","# 15. FINAL REPORT\n","# ============================================================\n","\n","total_removed = (\n"," stats[\"keyword_removed\"]\n"," + stats[\"short_removed\"]\n"," + stats[\"invalid_removed\"]\n"," + stats[\"duplicates_removed\"]\n",")\n","\n","print(\"\\n\" + \"=\" * 70)\n","print(\"FINAL CLEANING REPORT\")\n","print(\"=\" * 70)\n","\n","print(f\"Original items: {stats['original']:,}\")\n","\n","print(f\"Removed for forbidden words: {stats['keyword_removed']:,}\")\n","\n","print(f\"Removed for short text: {stats['short_removed']:,}\")\n","\n","print(f\"Removed for invalid text: {stats['invalid_removed']:,}\")\n","\n","print(f\"Removed duplicates: {stats['duplicates_removed']:,}\")\n","\n","print(\"-\" * 70)\n","\n","print(f\"Total removed: {total_removed:,}\")\n","\n","print(f\"Retained items: {stats['retained']:,}\")\n","\n","print(f\"Cleaned text files: {stats['cleaned_texts']:,}\")\n","\n","print(f\"Encrypted files: {encrypted_count:,}\")\n","\n","print(f\"Verified files: {verified_count:,}\")\n","\n","print(\"=\" * 70)\n","\n","\n","# ============================================================\n","# 16. UPLOAD AND COMMIT\n","# ============================================================\n","\n","# Upload is deliberately performed only after all\n","# validation and encryption checks have passed.\n","\n","if (\n"," stats[\"retained\"] == 0\n"," or encrypted_count != stats[\"retained\"] * 3\n"," or verified_count != encrypted_count\n","):\n"," raise RuntimeError(\n"," \"Final safety checks failed. Upload aborted.\"\n"," )\n","\n","print(\"\\nUploading cleaned dataset to Hugging Face...\")\n","\n","commit_info = api.upload_file(\n"," path_or_fileobj=str(OUTPUT_ARCHIVE),\n"," path_in_repo=ARCHIVE_NAME,\n"," repo_id=REPO_ID,\n"," repo_type=REPO_TYPE,\n"," commit_message=(\n"," \"Filter and clean training dataset; \"\n"," \"remove excluded keywords and short descriptions; \"\n"," \"normalize leading characters and capitalization; \"\n"," \"deduplicate cleaned outputs; \"\n"," \"re-encrypt complete dataset\"\n"," )\n",")\n","\n","print(\"\\n\" + \"=\" * 70)\n","print(\"KROMOSS CLEANING AND UPLOAD SUCCESSFUL\")\n","print(\"=\" * 70)\n","\n","print(\"Repository:\", REPO_ID)\n","print(\"Archive:\", ARCHIVE_NAME)\n","print(\"Final items:\", stats[\"retained\"])\n","print(\"Removed items:\", total_removed)\n","print(\"Encrypted files:\", encrypted_count)\n","print(\"Verified files:\", verified_count)\n","\n","print(\"\\nCommit information:\")\n","print(commit_info)\n","\n","print(\"\\nUpdated archive:\")\n","print(\n"," f\"https://huggingface.co/\"\n"," f\"{REPO_ID}/blob/main/{ARCHIVE_NAME}\"\n",")\n","\n","print(\"\\nAll operations completed successfully.\")\n"]}]}