Upload preprocess_7z_file_for_gemma4_training.ipynb

gemma4_training/preprocess_7z_file_for_gemma4_training.ipynb

@@ -0,0 +1 @@
+{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyOUIRksMt493fxnVXIo32Hn"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"version_major":2,"version_minor":0,"state":{"e8b67acbcf614769ae838ca733faecf2":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_61a46d577e2e431d97cb1c2fa6942d5d","IPY_MODEL_3fcd3928476e408b9ecb2810bd3c7923","IPY_MODEL_9f5fa485afff47ba93f096bb2af6b90d"],"layout":"IPY_MODEL_bc4f4288d7e64a5a9bc581a28aafc806"}},"61a46d577e2e431d97cb1c2fa6942d5d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_c435f730dfd54f56815b596980d68839","placeholder":"​","style":"IPY_MODEL_a50f8b2430d846c18c80a28530762a24","value":"gemma_train_set.7z: reconstructing file: 100%"}},"3fcd3928476e408b9ecb2810bd3c7923":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_d67ffce06c734c62a29f75b3ec43cdaf","max":4308892624,"min":0,"orientation":"horizontal","style":"IPY_MODEL_1c208c7370d44650a79be14cf8d8268a","value":4308892624}},"9f5fa485afff47ba93f096bb2af6b90d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_7acfcf7745dc4fa7968a946fe1edfcec","placeholder":"​","style":"IPY_MODEL_38d1bdddaa564047bfc8c8137aae3136","value":" 4.31GB / 4.31GB,  102MB/s  "}},"bc4f4288d7e64a5a9bc581a28aafc806":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c435f730dfd54f56815b596980d68839":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a50f8b2430d846c18c80a28530762a24":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"d67ffce06c734c62a29f75b3ec43cdaf":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1c208c7370d44650a79be14cf8d8268a":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"7acfcf7745dc4fa7968a946fe1edfcec":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"38d1bdddaa564047bfc8c8137aae3136":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"cfdb205289204c70a4a804289a270bda":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_8f822c4a63684dd59ded7f51dbdf298a","IPY_MODEL_ca7d63c3466847bc9c5eccd4d4406324","IPY_MODEL_fed49aa0e05c4490aa18de76ef2a9738"],"layout":"IPY_MODEL_07fe78c9a2ce4932ab6d97a51ebeb67b"}},"8f822c4a63684dd59ded7f51dbdf298a":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d00269e2470a43ec8b48352453baad75","placeholder":"​","style":"IPY_MODEL_092b7c336f4d48968b3b016de4837ba5","value":"gemma_train_set.7z: downloading bytes: "}},"ca7d63c3466847bc9c5eccd4d4406324":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_e245e384e8ae4da7b266eb9c38b37bb9","max":4308892624,"min":0,"orientation":"horizontal","style":"IPY_MODEL_94935c3ecea849e09ebf871308918ef0","value":4308892624}},"fed49aa0e05c4490aa18de76ef2a9738":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_2f063bf60dbc4afaa9a246ad84edf9ee","placeholder":"​","style":"IPY_MODEL_a22c761448754eb6aea7e0f4bdb3bc36","value":" 4.31GB, 73.6MB/s  "}},"07fe78c9a2ce4932ab6d97a51ebeb67b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d00269e2470a43ec8b48352453baad75":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"092b7c336f4d48968b3b016de4837ba5":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"e245e384e8ae4da7b266eb9c38b37bb9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"94935c3ecea849e09ebf871308918ef0":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"2f063bf60dbc4afaa9a246ad84edf9ee":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a22c761448754eb6aea7e0f4bdb3bc36":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a395cba8cd3345b2b10a0eca20e021ef":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_30bebdccc00f4c26ad39240aa96471c6","IPY_MODEL_f65467f34c2142dda0bc884bbc654b46","IPY_MODEL_27c095f57bd949f4a7981e239b5cc5fe"],"layout":"IPY_MODEL_bd64c1c903a348dd9993dd76c842cd11"}},"30bebdccc00f4c26ad39240aa96471c6":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_786926e91d72478097074303ca3d3f30","placeholder":"​","style":"IPY_MODEL_1da0ec785e584daa9e1f4965d05d8b75","value":"Converting examples: 100%"}},"f65467f34c2142dda0bc884bbc654b46":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_3c442fecd2e842858a492f0cec2dfd93","max":12655,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c2a2c0f84304431ba4dd1b9bb304dbf4","value":12655}},"27c095f57bd949f4a7981e239b5cc5fe":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_5e2a38b314254c68a639c316b55987a1","placeholder":"​","style":"IPY_MODEL_702672013ced43ddb98b1938453553a1","value":" 12655/12655 [59:28&lt;00:00,  3.85it/s]"}},"bd64c1c903a348dd9993dd76c842cd11":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"786926e91d72478097074303ca3d3f30":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1da0ec785e584daa9e1f4965d05d8b75":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"3c442fecd2e842858a492f0cec2dfd93":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c2a2c0f84304431ba4dd1b9bb304dbf4":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"5e2a38b314254c68a639c316b55987a1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"702672013ced43ddb98b1938453553a1":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"66bd352026c74894ab7c247845097ec1":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_c0f12acc2134429c8da48fd811b8a45f","IPY_MODEL_4453d70100a04154ae0944505fdc2057","IPY_MODEL_fa62d27dac5e4ff49d44c4010439d707"],"layout":"IPY_MODEL_356701ff8deb44ebb225257c53bffb99"}},"c0f12acc2134429c8da48fd811b8a45f":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_f416792813b6467da0fcc2ee4db79859","placeholder":"​","style":"IPY_MODEL_1974a161c1f74a36ac1c175bd9432e90","value":"Saving the dataset (7/7 shards): 100%"}},"4453d70100a04154ae0944505fdc2057":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_87463aa2ffb94e68a5e749d622d5a08a","max":12655,"min":0,"orientation":"horizontal","style":"IPY_MODEL_f2b7fa6ffa8f418495a29aceb169ec6f","value":12655}},"fa62d27dac5e4ff49d44c4010439d707":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8aee83c969dd4ebd8a9ac4e265c35c0e","placeholder":"​","style":"IPY_MODEL_1735d87934c34a0ba47c729721e5d733","value":" 12655/12655 [00:52&lt;00:00, 320.01 examples/s]"}},"356701ff8deb44ebb225257c53bffb99":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f416792813b6467da0fcc2ee4db79859":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1974a161c1f74a36ac1c175bd9432e90":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"87463aa2ffb94e68a5e749d622d5a08a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f2b7fa6ffa8f418495a29aceb169ec6f":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"8aee83c969dd4ebd8a9ac4e265c35c0e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1735d87934c34a0ba47c729721e5d733":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}}},"cells":[{"cell_type":"code","source":["# ============================================================\n","# GEMMA 4 ENCRYPTED MULTIMODAL DATASET CONVERTER\n","#\n","# SOURCE:\n","#   codeShare/kromoss-lora-exports / gemma_train_set.7z\n","#\n","# OUTPUT:\n","#   Encrypted Hugging Face / Apache Arrow dataset\n","#\n","# IMPORTANT FIXES:\n","#   - Uses ArrowWriter instead of Dataset.from_generator()\n","#   - Avoids pickling AESGCM and cryptography objects\n","#   - Writes records incrementally\n","#   - Fixes decimal filtering\n","#   - Removes stray punctuation artifacts\n","#   - Cleans both input and output text\n","#   - Preserves ordinary integers\n","#   - Encrypts images and conversations independently\n","# ============================================================\n","\n","\n","# ============================================================\n","# 1. INSTALL DEPENDENCIES\n","# ============================================================\n","\n","!pip install -q -U datasets pyarrow py7zr cryptography pillow huggingface_hub tqdm\n","\n","\n","# ============================================================\n","# 2. IMPORTS\n","# ============================================================\n","\n","import os\n","import io\n","import gc\n","import re\n","import json\n","import time\n","import shutil\n","import secrets\n","import base64\n","import getpass\n","\n","from pathlib import Path\n","\n","import py7zr\n","\n","from PIL import Image, ImageOps\n","\n","from datasets import Dataset, Features, Value\n","from datasets.arrow_writer import ArrowWriter\n","\n","from huggingface_hub import HfApi, hf_hub_download\n","\n","from cryptography.hazmat.primitives.ciphers.aead import AESGCM\n","from cryptography.hazmat.primitives.kdf.scrypt import Scrypt\n","\n","from tqdm.auto import tqdm\n","\n","\n","# ============================================================\n","# 3. CONFIGURATION\n","# ============================================================\n","\n","SOURCE_REPO = \"codeShare/kromoss-lora-exports\"\n","\n","SOURCE_ARCHIVE = \"gemma_train_set.7z\"\n","\n","SOURCE_REVISION = \"main\"\n","\n","REPO_TYPE = \"model\"\n","\n","DESTINATION_SUBFOLDER = \"gemma4_arrow_encrypted_v1\"\n","\n","\n","# ------------------------------------------------------------\n","# Image preprocessing\n","# ------------------------------------------------------------\n","\n","MAX_IMAGE_DIMENSION = 1024\n","\n","JPEG_QUALITY = 95\n","\n","WEBP_QUALITY = 95\n","\n","\n","# ------------------------------------------------------------\n","# Arrow configuration\n","# ------------------------------------------------------------\n","\n","ARROW_MAX_SHARD_SIZE = \"512MB\"\n","\n","ARROW_WRITER_BATCH_SIZE = 16\n","\n","\n","# ------------------------------------------------------------\n","# Conversion configuration\n","# ------------------------------------------------------------\n","\n","MAX_EXAMPLES = 0\n","\n","# 0 = process everything.\n","# 100 = process only the first 100 examples.\n","\n","\n","# ------------------------------------------------------------\n","# Upload configuration\n","# ------------------------------------------------------------\n","\n","ALLOW_OVERWRITE = False\n","\n","# If the destination folder already exists, the script stops.\n","# Use a new DESTINATION_SUBFOLDER for a new version.\n","\n","\n","# ------------------------------------------------------------\n","# PCENC1 source encryption\n","# ------------------------------------------------------------\n","\n","PCENC_MAGIC = b\"PCENC1\"\n","\n","\n","# ------------------------------------------------------------\n","# Output encryption\n","# ------------------------------------------------------------\n","\n","OUTPUT_CIPHER = \"AES-256-GCM\"\n","\n","OUTPUT_SCRYPT_N = 2**14\n","\n","OUTPUT_SCRYPT_R = 8\n","\n","OUTPUT_SCRYPT_P = 1\n","\n","\n","# ------------------------------------------------------------\n","# Working directories\n","# ------------------------------------------------------------\n","\n","WORK_DIR = Path(\"/content/gemma4_conversion\")\n","\n","DOWNLOAD_DIR = WORK_DIR / \"downloads\"\n","\n","EXTRACT_DIR = WORK_DIR / \"extracted\"\n","\n","OUTPUT_DIR = WORK_DIR / \"arrow_dataset\"\n","\n","CACHE_DIR = WORK_DIR / \"arrow_build_cache\"\n","\n","TEMP_ARROW_PATH = WORK_DIR / \"encrypted_dataset.arrow\"\n","\n","for directory in [\n","    WORK_DIR,\n","    DOWNLOAD_DIR,\n","    EXTRACT_DIR,\n","    CACHE_DIR\n","]:\n","    directory.mkdir(\n","        parents=True,\n","        exist_ok=True\n","    )\n","\n","\n","# ============================================================\n","# 4. AUTHENTICATION AND PASSWORDS\n","# ============================================================\n","\n","def get_colab_secret(name):\n","\n","    try:\n","\n","        from google.colab import userdata\n","\n","        return userdata.get(name)\n","\n","    except Exception:\n","\n","        return None\n","\n","\n","HF_TOKEN = (\n","    os.environ.get(\"HF_TOKEN\")\n","    or get_colab_secret(\"HF_TOKEN\")\n",")\n","\n","if not HF_TOKEN:\n","\n","    HF_TOKEN = getpass.getpass(\n","        \"Hugging Face write token: \"\n","    )\n","\n","if not HF_TOKEN:\n","\n","    raise RuntimeError(\n","        \"Hugging Face token is required.\"\n","    )\n","\n","\n","# Ask for the source archive password.\n","SOURCE_PASSWORD = 'blueberry'\n","\n","#getpass.getpass(\n","#    \"Source archive / PCENC1 password: \"\n","#)\n","\n","if not SOURCE_PASSWORD:\n","\n","    raise RuntimeError(\n","        \"Source password is required.\"\n","    )\n","\n","\n","# Ask for a separate output encryption password.\n","OUTPUT_PASSWORD = 'blueberry'\n","\n","#getpass.getpass(\n","#    \"New output dataset encryption password: \"\n","#)\n","\n","if not OUTPUT_PASSWORD:\n","\n","    raise RuntimeError(\n","        \"Output encryption password is required.\"\n","    )\n","\n","\n","api = HfApi(token=HF_TOKEN)\n","\n","account = api.whoami()\n","\n","print(\"Authenticated:\", account[\"name\"])\n","\n","\n","# ============================================================\n","# 5. TEXT FILTER CONFIGURATION\n","# ============================================================\n","\n","TEXT_PATTERNS = [\n","\n","    # Quality-related terms.\n","\n","    re.compile(\n","        r\"\\bmasterpiece\\b\",\n","        re.IGNORECASE\n","    ),\n","\n","    re.compile(\n","        r\"\\bbest\\s+quality\\b\",\n","        re.IGNORECASE\n","    ),\n","\n","    re.compile(\n","        r\"\\bhighres\\b\",\n","        re.IGNORECASE\n","    ),\n","\n","    re.compile(\n","        r\"\\babsurdres\\b\",\n","        re.IGNORECASE\n","    ),\n","\n","    # Score tags:\n","    # score1\n","    # score_1\n","    # score 1\n","    # score99\n","    # score_99\n","    # score 99\n","\n","    re.compile(\n","        r\"\\bscore(?:_|\\s*)\\d+\\b\",\n","        re.IGNORECASE\n","    ),\n","\n","    # Decimal numbers:\n","    # 1.6\n","    # 0.003\n","    # 123.456\n","    #\n","    # Allows sentence punctuation after the decimal.\n","    # Does not remove ordinary integers.\n","\n","    re.compile(\n","        r\"(?<![\\w.])[+-]?\\d+\\.\\d+(?!\\w)\"\n","    )\n","\n","]\n","\n","\n","def clean_training_text(text):\n","\n","    if not isinstance(text, str):\n","\n","        raise TypeError(\n","            \"Training text must be a string.\"\n","        )\n","\n","    # --------------------------------------------------------\n","    # 1. REMOVE TERMS, SCORE TAGS AND DECIMAL NUMBERS\n","    # --------------------------------------------------------\n","\n","    for pattern in TEXT_PATTERNS:\n","\n","        text = pattern.sub(\n","            \" \",\n","            text\n","        )\n","\n","    # --------------------------------------------------------\n","    # 2. REMOVE REPEATED COMMA/PERIOD ARTIFACTS\n","    # --------------------------------------------------------\n","\n","    # Remove repeated punctuation attached to words:\n","    # word... -> word\n","    # word,,, -> word\n","    # word.,  -> word\n","\n","    text = re.sub(\n","        r\"(?<=\\w)[,.]{2,}\",\n","        \"\",\n","        text\n","    )\n","\n","    # Remove punctuation-only tokens:\n","    # ,,,  ...  .,  ,.  . , .\n","    #\n","    # This does not remove normal punctuation attached\n","    # directly to words, such as \"Hello, world.\"\n","\n","    text = re.sub(\n","        r\"(?<!\\w)[,.]+(?!\\w)\",\n","        \" \",\n","        text\n","    )\n","\n","    # --------------------------------------------------------\n","    # 3. CLEAN SPACING\n","    # --------------------------------------------------------\n","\n","    # Normalize spaces and tabs.\n","    text = re.sub(\n","        r\"[ \\t]+\",\n","        \" \",\n","        text\n","    )\n","\n","    # Remove spaces before ordinary punctuation.\n","    text = re.sub(\n","        r\"\\s+([,;:!?])\",\n","        r\"\\1\",\n","        text\n","    )\n","\n","    # Remove repeated blank lines.\n","    text = re.sub(\n","        r\"\\n[ \\t]*\\n(?:[ \\t]*\\n)+\",\n","        \"\\n\\n\",\n","        text\n","    )\n","\n","    # Remove leading and trailing whitespace.\n","    text = text.strip()\n","\n","    return text\n","\n","\n","# ============================================================\n","# 6. TEST TEXT FILTER\n","# ============================================================\n","\n","test_text = \"\"\"\n","Masterpiece, best quality, highres, absurdres.\n","\n","SCORE_1 score2 score 3 score99.\n","\n","This image has dimensions 1.6 and 0.003.\n","\n","The integer 95 should remain.\n","\n","The normal text should remain intact.\n","\n","This is,,, another test.\n","\n","This is... another test.\n","\n","This is ., another test.\n","\n","Normal punctuation should remain: red, blue and green.\n","\"\"\"\n","\n","print(\"\\nTEXT FILTER TEST\")\n","\n","print(\"=\" * 70)\n","\n","print(\"BEFORE:\")\n","print(test_text)\n","\n","filtered_test = clean_training_text(test_text)\n","\n","print(\"\\nAFTER:\")\n","print(filtered_test)\n","\n","\n","# Verify forbidden terms are removed.\n","\n","assert not re.search(\n","    r\"\\bmasterpiece\\b|\\bbest\\s+quality\\b|\\bhighres\\b|\\babsurdres\\b\",\n","    filtered_test,\n","    re.IGNORECASE\n",")\n","\n","assert not re.search(\n","    r\"\\bscore(?:_|\\s*)\\d+\\b\",\n","    filtered_test,\n","    re.IGNORECASE\n",")\n","\n","assert not re.search(\n","    r\"(?<![\\w.])[+-]?\\d+\\.\\d+(?!\\w)\",\n","    filtered_test\n",")\n","\n","assert \"95\" in filtered_test\n","\n","assert \"red, blue\" in filtered_test\n","\n","assert \"green.\" in filtered_test\n","\n","assert not re.search(\n","    r\"(?<!\\w)[,.]+(?!\\w)\",\n","    filtered_test\n",")\n","\n","assert not re.search(\n","    r\"(?<=\\w)[,.]{2,}\",\n","    filtered_test\n",")\n","\n","print(\"\\nAll text filter tests passed.\")\n","\n","\n","# ============================================================\n","# 7. DOWNLOAD ORIGINAL ARCHIVE\n","# ============================================================\n","\n","archive_path = DOWNLOAD_DIR / SOURCE_ARCHIVE\n","\n","if archive_path.exists():\n","\n","    print(\n","        \"\\nExisting downloaded archive found. Reusing it.\"\n","    )\n","\n","else:\n","\n","    print(\"\\nDownloading original dataset...\")\n","\n","    downloaded_path = hf_hub_download(\n","\n","        repo_id=SOURCE_REPO,\n","\n","        repo_type=REPO_TYPE,\n","\n","        filename=SOURCE_ARCHIVE,\n","\n","        revision=SOURCE_REVISION,\n","\n","        token=HF_TOKEN,\n","\n","        local_dir=str(DOWNLOAD_DIR)\n","\n","    )\n","\n","    archive_path = Path(downloaded_path)\n","\n","\n","print(\"Archive:\", archive_path)\n","\n","print(\n","    \"Archive size:\",\n","    f\"{archive_path.stat().st_size / 1024**3:.2f} GB\"\n",")\n","\n","\n","# ============================================================\n","# 8. EXTRACT ORIGINAL 7Z\n","# ============================================================\n","\n","if EXTRACT_DIR.exists():\n","\n","    existing_extracted_files = list(\n","        EXTRACT_DIR.iterdir()\n","    )\n","\n","else:\n","\n","    existing_extracted_files = []\n","\n","\n","if existing_extracted_files:\n","\n","    print(\n","        \"\\nExisting extraction found. Reusing extracted files.\"\n","    )\n","\n","else:\n","\n","    print(\"\\nExtracting archive...\")\n","\n","    with py7zr.SevenZipFile(\n","\n","        archive_path,\n","\n","        mode=\"r\",\n","\n","        password=SOURCE_PASSWORD\n","\n","    ) as archive:\n","\n","        archive.extractall(\n","            path=EXTRACT_DIR\n","        )\n","\n","    print(\"Extraction completed.\")\n","\n","\n","# ============================================================\n","# 9. PCENC1 DECRYPTION\n","# ============================================================\n","\n","def derive_source_key(password, salt):\n","\n","    return Scrypt(\n","\n","        salt=salt,\n","\n","        length=32,\n","\n","        n=2**14,\n","\n","        r=8,\n","\n","        p=1\n","\n","    ).derive(\n","        password.encode(\"utf-8\")\n","    )\n","\n","\n","def decrypt_pcenc1(blob):\n","\n","    if not blob.startswith(PCENC_MAGIC):\n","\n","        return blob\n","\n","    # Header layout:\n","    # 0:6   = PCENC1\n","    # 6:22  = salt\n","    # 22:34 = nonce\n","    # 34:   = ciphertext + authentication tag\n","\n","    if len(blob) < 50:\n","\n","        raise ValueError(\n","            \"Truncated PCENC1 payload.\"\n","        )\n","\n","    salt = blob[6:22]\n","\n","    nonce = blob[22:34]\n","\n","    ciphertext = blob[34:]\n","\n","    key = derive_source_key(\n","        SOURCE_PASSWORD,\n","        salt\n","    )\n","\n","    return AESGCM(key).decrypt(\n","\n","        nonce,\n","\n","        ciphertext,\n","\n","        PCENC_MAGIC\n","\n","    )\n","\n","\n","# ============================================================\n","# 10. IMAGE PREPROCESSING\n","# ============================================================\n","\n","IMAGE_EXTENSIONS = {\n","\n","    \".jpg\",\n","    \".jpeg\",\n","    \".png\",\n","    \".webp\",\n","    \".bmp\",\n","    \".tif\",\n","    \".tiff\"\n","\n","}\n","\n","\n","def preprocess_image(image_bytes):\n","\n","    with Image.open(\n","        io.BytesIO(image_bytes)\n","    ) as original:\n","\n","        # Correct EXIF orientation.\n","        image = ImageOps.exif_transpose(\n","            original\n","        )\n","\n","        # ----------------------------------------------------\n","        # Convert to RGB.\n","        # Transparent areas become white.\n","        # ----------------------------------------------------\n","\n","        if (\n","\n","            image.mode in (\"RGBA\", \"LA\")\n","\n","            or \"transparency\" in image.info\n","\n","        ):\n","\n","            rgba = image.convert(\"RGBA\")\n","\n","            background = Image.new(\n","\n","                \"RGBA\",\n","\n","                rgba.size,\n","\n","                (255, 255, 255, 255)\n","\n","            )\n","\n","            background.alpha_composite(rgba)\n","\n","            image = background.convert(\"RGB\")\n","\n","        else:\n","\n","            image = image.convert(\"RGB\")\n","\n","\n","        # ----------------------------------------------------\n","        # Resize to maximum dimension of 1024.\n","        # ----------------------------------------------------\n","\n","        width, height = image.size\n","\n","        largest = max(\n","            width,\n","            height\n","        )\n","\n","        if largest > MAX_IMAGE_DIMENSION:\n","\n","            scale = (\n","                MAX_IMAGE_DIMENSION / largest\n","            )\n","\n","            new_width = max(\n","                1,\n","                round(width * scale)\n","            )\n","\n","            new_height = max(\n","                1,\n","                round(height * scale)\n","            )\n","\n","            image = image.resize(\n","\n","                (new_width, new_height),\n","\n","                Image.Resampling.LANCZOS\n","\n","            )\n","\n","\n","        # ----------------------------------------------------\n","        # Select output format.\n","        # ----------------------------------------------------\n","\n","        if original.format == \"PNG\":\n","\n","            image_format = \"PNG\"\n","\n","            save_options = {\n","\n","                \"format\": \"PNG\",\n","\n","                \"optimize\": True\n","\n","            }\n","\n","        elif original.format == \"WEBP\":\n","\n","            image_format = \"WEBP\"\n","\n","            save_options = {\n","\n","                \"format\": \"WEBP\",\n","\n","                \"quality\": WEBP_QUALITY,\n","\n","                \"method\": 6\n","\n","            }\n","\n","        else:\n","\n","            image_format = \"JPEG\"\n","\n","            save_options = {\n","\n","                \"format\": \"JPEG\",\n","\n","                \"quality\": JPEG_QUALITY,\n","\n","                \"optimize\": True,\n","\n","                \"subsampling\": 0\n","\n","            }\n","\n","\n","        output = io.BytesIO()\n","\n","        image.save(\n","            output,\n","            **save_options\n","        )\n","\n","        return {\n","\n","            \"bytes\": output.getvalue(),\n","\n","            \"width\": image.width,\n","\n","            \"height\": image.height,\n","\n","            \"format\": image_format\n","\n","        }\n","\n","\n","# ============================================================\n","# 11. OUTPUT ENCRYPTION\n","# ============================================================\n","\n","DATASET_SALT = secrets.token_bytes(16)\n","\n","OUTPUT_KEY = Scrypt(\n","\n","    salt=DATASET_SALT,\n","\n","    length=32,\n","\n","    n=OUTPUT_SCRYPT_N,\n","\n","    r=OUTPUT_SCRYPT_R,\n","\n","    p=OUTPUT_SCRYPT_P\n","\n",").derive(\n","    OUTPUT_PASSWORD.encode(\"utf-8\")\n",")\n","\n","\n","def encrypt_payload(\n","\n","    plaintext,\n","\n","    sample_id,\n","\n","    field_name\n","\n","):\n","\n","    nonce = secrets.token_bytes(12)\n","\n","    aad = (\n","\n","        f\"gemma4-v1:{sample_id}:{field_name}\"\n","\n","    ).encode(\"utf-8\")\n","\n","    ciphertext = AESGCM(OUTPUT_KEY).encrypt(\n","\n","        nonce,\n","\n","        plaintext,\n","\n","        aad\n","\n","    )\n","\n","    return nonce + ciphertext\n","\n","\n","def decrypt_payload(\n","\n","    encrypted_payload,\n","\n","    sample_id,\n","\n","    field_name\n","\n","):\n","\n","    nonce = encrypted_payload[:12]\n","\n","    ciphertext = encrypted_payload[12:]\n","\n","    aad = (\n","\n","        f\"gemma4-v1:{sample_id}:{field_name}\"\n","\n","    ).encode(\"utf-8\")\n","\n","    return AESGCM(OUTPUT_KEY).decrypt(\n","\n","        nonce,\n","\n","        ciphertext,\n","\n","        aad\n","\n","    )\n","\n","\n","# ============================================================\n","# 12. DISCOVER TRAINING EXAMPLES\n","# ============================================================\n","\n","print(\"\\nDiscovering training examples...\")\n","\n","examples = []\n","\n","for image_path in EXTRACT_DIR.rglob(\"*\"):\n","\n","    if not image_path.is_file():\n","\n","        continue\n","\n","    if image_path.suffix.lower() not in IMAGE_EXTENSIONS:\n","\n","        continue\n","\n","    stem = image_path.stem\n","\n","    input_path = image_path.with_name(\n","        stem + \"_input.txt\"\n","    )\n","\n","    output_path = image_path.with_name(\n","        stem + \"_output.txt\"\n","    )\n","\n","    if (\n","\n","        input_path.is_file()\n","\n","        and output_path.is_file()\n","\n","    ):\n","\n","        examples.append({\n","\n","            \"image\": image_path,\n","\n","            \"input\": input_path,\n","\n","            \"output\": output_path\n","\n","        })\n","\n","\n","examples.sort(\n","\n","    key=lambda row: str(\n","\n","        row[\"image\"].relative_to(EXTRACT_DIR)\n","\n","    )\n","\n",")\n","\n","print(\"Discovered examples:\", len(examples))\n","\n","if not examples:\n","\n","    raise RuntimeError(\n","        \"No valid image/input/output triplets found.\"\n","    )\n","\n","\n","if MAX_EXAMPLES > 0:\n","\n","    examples = examples[:MAX_EXAMPLES]\n","\n","\n","print(\"Examples to convert:\", len(examples))\n","\n","\n","# ============================================================\n","# 13. ARROW SCHEMA\n","# ============================================================\n","\n","FEATURES = Features({\n","\n","    \"sample_id\": Value(\"string\"),\n","\n","    \"image_enc\": Value(\"binary\"),\n","\n","    \"conversation_enc\": Value(\"binary\"),\n","\n","    \"width\": Value(\"int32\"),\n","\n","    \"height\": Value(\"int32\"),\n","\n","    \"image_format\": Value(\"string\")\n","\n","})\n","\n","\n","# ============================================================\n","# 14. CONVERSION STATISTICS\n","# ============================================================\n","\n","conversion_stats = {\n","\n","    \"processed\": 0,\n","\n","    \"skipped\": 0,\n","\n","    \"text_changed\": 0,\n","\n","    \"input_text_changed\": 0,\n","\n","    \"output_text_changed\": 0,\n","\n","    \"errors\": []\n","\n","}\n","\n","\n","# ============================================================\n","# 15. PREPARE ARROW WRITER\n","# ============================================================\n","\n","if TEMP_ARROW_PATH.exists():\n","\n","    TEMP_ARROW_PATH.unlink()\n","\n","\n","writer = ArrowWriter(\n","\n","    path=str(TEMP_ARROW_PATH),\n","\n","    features=FEATURES,\n","\n","    writer_batch_size=ARROW_WRITER_BATCH_SIZE\n","\n",")\n","\n","\n","# ============================================================\n","# 16. CONVERT AND WRITE RECORDS DIRECTLY TO ARROW\n","#\n","# IMPORTANT:\n","# This replaces Dataset.from_generator().\n","#\n","# ArrowWriter writes incrementally and does not pickle\n","# AESGCM, OUTPUT_KEY, or the generator's global variables.\n","# ============================================================\n","\n","print(\"\\n\" + \"=\" * 75)\n","\n","print(\"BUILDING CLEANED ENCRYPTED ARROW DATASET\")\n","\n","print(\"=\" * 75)\n","\n","\n","for index, row in enumerate(\n","\n","    tqdm(\n","        examples,\n","        desc=\"Converting examples\"\n","    )\n","\n","):\n","\n","    sample_id = f\"{index:08d}\"\n","\n","    try:\n","\n","        # ----------------------------------------------------\n","        # A. DECRYPT INPUT TEXT\n","        # ----------------------------------------------------\n","\n","        prompt_bytes = decrypt_pcenc1(\n","\n","            row[\"input\"].read_bytes()\n","\n","        )\n","\n","        prompt = prompt_bytes.decode(\n","            \"utf-8-sig\"\n","        )\n","\n","\n","        # ----------------------------------------------------\n","        # B. DECRYPT EXPECTED OUTPUT\n","        # ----------------------------------------------------\n","\n","        answer_bytes = decrypt_pcenc1(\n","\n","            row[\"output\"].read_bytes()\n","\n","        )\n","\n","        answer = answer_bytes.decode(\n","            \"utf-8-sig\"\n","        )\n","\n","\n","        # ----------------------------------------------------\n","        # C. CLEAN INPUT AND OUTPUT TEXT\n","        # ----------------------------------------------------\n","\n","        original_prompt = prompt\n","\n","        original_answer = answer\n","\n","        prompt = clean_training_text(\n","            prompt\n","        )\n","\n","        answer = clean_training_text(\n","            answer\n","        )\n","\n","\n","        prompt_changed = (\n","            prompt != original_prompt.strip()\n","        )\n","\n","        answer_changed = (\n","            answer != original_answer.strip()\n","        )\n","\n","\n","        if prompt_changed:\n","\n","            conversion_stats[\n","                \"input_text_changed\"\n","            ] += 1\n","\n","\n","        if answer_changed:\n","\n","            conversion_stats[\n","                \"output_text_changed\"\n","            ] += 1\n","\n","\n","        if prompt_changed or answer_changed:\n","\n","            conversion_stats[\n","                \"text_changed\"\n","            ] += 1\n","\n","\n","        # Reject empty text after filtering.\n","        if not prompt:\n","\n","            raise ValueError(\n","                \"Input prompt empty after cleaning.\"\n","            )\n","\n","        if not answer:\n","\n","            raise ValueError(\n","                \"Expected response empty after cleaning.\"\n","            )\n","\n","\n","        # ----------------------------------------------------\n","        # D. DECRYPT AND PREPROCESS IMAGE\n","        # ----------------------------------------------------\n","\n","        encrypted_or_plain_image = row[\"image\"].read_bytes()\n","\n","        image_bytes = decrypt_pcenc1(\n","            encrypted_or_plain_image\n","        )\n","\n","        processed = preprocess_image(\n","            image_bytes\n","        )\n","\n","\n","        # ----------------------------------------------------\n","        # E. CONSTRUCT GEMMA CONVERSATION\n","        # ----------------------------------------------------\n","\n","        conversation = {\n","\n","            \"messages\": [\n","\n","                {\n","\n","                    \"role\": \"user\",\n","\n","                    \"content\": [\n","\n","                        {\n","\n","                            \"type\": \"image\"\n","\n","                        },\n","\n","                        {\n","\n","                            \"type\": \"text\",\n","\n","                            \"text\": prompt\n","\n","                        }\n","\n","                    ]\n","\n","                },\n","\n","                {\n","\n","                    \"role\": \"assistant\",\n","\n","                    \"content\": [\n","\n","                        {\n","\n","                            \"type\": \"text\",\n","\n","                            \"text\": answer\n","\n","                        }\n","\n","                    ]\n","\n","                }\n","\n","            ]\n","\n","        }\n","\n","\n","        conversation_bytes = json.dumps(\n","\n","            conversation,\n","\n","            ensure_ascii=False,\n","\n","            separators=(\",\", \":\")\n","\n","        ).encode(\"utf-8\")\n","\n","\n","        # ----------------------------------------------------\n","        # F. ENCRYPT IMAGE\n","        # ----------------------------------------------------\n","\n","        encrypted_image = encrypt_payload(\n","\n","            processed[\"bytes\"],\n","\n","            sample_id,\n","\n","            \"image\"\n","\n","        )\n","\n","\n","        # ----------------------------------------------------\n","        # G. ENCRYPT CONVERSATION\n","        # ----------------------------------------------------\n","\n","        encrypted_conversation = encrypt_payload(\n","\n","            conversation_bytes,\n","\n","            sample_id,\n","\n","            \"conversation\"\n","\n","        )\n","\n","\n","        # ----------------------------------------------------\n","        # H. WRITE RECORD DIRECTLY TO ARROW\n","        # ----------------------------------------------------\n","\n","        arrow_record = {\n","\n","            \"sample_id\": sample_id,\n","\n","            \"image_enc\": encrypted_image,\n","\n","            \"conversation_enc\": encrypted_conversation,\n","\n","            \"width\": processed[\"width\"],\n","\n","            \"height\": processed[\"height\"],\n","\n","            \"image_format\": processed[\"format\"]\n","\n","        }\n","\n","        writer.write(arrow_record)\n","\n","        conversion_stats[\"processed\"] += 1\n","\n","\n","        # Periodic garbage collection.\n","        if conversion_stats[\"processed\"] % 500 == 0:\n","\n","            gc.collect()\n","\n","\n","    except Exception as exc:\n","\n","        conversion_stats[\"skipped\"] += 1\n","\n","        conversion_stats[\"errors\"].append({\n","\n","            \"sample_id\": sample_id,\n","\n","            \"error\": repr(exc)\n","\n","        })\n","\n","        print(\n","            f\"\\nERROR {sample_id}: {repr(exc)}\"\n","        )\n","\n","        continue\n","\n","\n","# ============================================================\n","# 17. FINALIZE ARROW FILE\n","# ============================================================\n","\n","print(\"\\nFinalizing Arrow writer...\")\n","\n","writer.finalize()\n","\n","del writer\n","\n","gc.collect()\n","\n","if not TEMP_ARROW_PATH.exists():\n","\n","    raise RuntimeError(\n","        \"Arrow file was not created.\"\n","    )\n","\n","if TEMP_ARROW_PATH.stat().st_size == 0:\n","\n","    raise RuntimeError(\n","        \"Arrow file is empty.\"\n","    )\n","\n","\n","print(\n","    \"Temporary Arrow file size:\",\n","    f\"{TEMP_ARROW_PATH.stat().st_size / 1024**3:.2f} GB\"\n",")\n","\n","print(\"Successfully processed:\", conversion_stats[\"processed\"])\n","\n","print(\"Skipped:\", conversion_stats[\"skipped\"])\n","\n","print(\"Modified samples:\", conversion_stats[\"text_changed\"])\n","\n","print(\"Modified prompts:\", conversion_stats[\"input_text_changed\"])\n","\n","print(\"Modified responses:\", conversion_stats[\"output_text_changed\"])\n","\n","\n","if conversion_stats[\"processed\"] == 0:\n","\n","    raise RuntimeError(\n","        \"No valid records were produced.\"\n","    )\n","\n","\n","# ============================================================\n","# 18. LOAD ARROW FILE AS HUGGING FACE DATASET\n","# ============================================================\n","\n","print(\"\\nLoading Arrow file as Hugging Face Dataset...\")\n","\n","dataset = Dataset.from_file(\n","    str(TEMP_ARROW_PATH)\n",")\n","\n","print(\"Arrow rows:\", len(dataset))\n","\n","if len(dataset) != conversion_stats[\"processed\"]:\n","\n","    raise RuntimeError(\n","        \"Arrow record count mismatch.\"\n","    )\n","\n","\n","# ============================================================\n","# 19. VERIFY OUTPUT ENCRYPTION\n","# ============================================================\n","\n","print(\"\\nVerifying encryption...\")\n","\n","test_row = dataset[0]\n","\n","\n","test_image = decrypt_payload(\n","\n","    test_row[\"image_enc\"],\n","\n","    test_row[\"sample_id\"],\n","\n","    \"image\"\n","\n",")\n","\n","\n","test_conversation_bytes = decrypt_payload(\n","\n","    test_row[\"conversation_enc\"],\n","\n","    test_row[\"sample_id\"],\n","\n","    \"conversation\"\n","\n",")\n","\n","\n","test_conversation = json.loads(\n","\n","    test_conversation_bytes.decode(\"utf-8\")\n","\n",")\n","\n","\n","# Verify image integrity.\n","\n","with Image.open(\n","    io.BytesIO(test_image)\n",") as image:\n","\n","    image.verify()\n","\n","\n","# Verify image dimensions.\n","\n","assert test_row[\"width\"] <= MAX_IMAGE_DIMENSION\n","\n","assert test_row[\"height\"] <= MAX_IMAGE_DIMENSION\n","\n","\n","# Verify conversation structure.\n","\n","assert len(test_conversation[\"messages\"]) == 2\n","\n","assert test_conversation[\"messages\"][0][\"role\"] == \"user\"\n","\n","assert test_conversation[\"messages\"][1][\"role\"] == \"assistant\"\n","\n","\n","# Verify text filtering.\n","\n","verified_text = json.dumps(\n","\n","    test_conversation,\n","\n","    ensure_ascii=False\n","\n",")\n","\n","\n","assert not re.search(\n","\n","    r\"\\bmasterpiece\\b|\\bbest\\s+quality\\b|\\bhighres\\b|\\babsurdres\\b\",\n","\n","    verified_text,\n","\n","    re.IGNORECASE\n","\n",")\n","\n","\n","assert not re.search(\n","\n","    r\"\\bscore(?:_|\\s*)\\d+\\b\",\n","\n","    verified_text,\n","\n","    re.IGNORECASE\n","\n",")\n","\n","\n","assert not re.search(\n","\n","    r\"(?<![\\w.])[+-]?\\d+\\.\\d+(?!\\w)\",\n","\n","    verified_text\n","\n",")\n","\n","\n","assert not re.search(\n","\n","    r\"(?<!\\w)[,.]+(?!\\w)\",\n","\n","    verified_text\n","\n",")\n","\n","\n","assert not re.search(\n","\n","    r\"(?<=\\w)[,.]{2,}\",\n","\n","    verified_text\n","\n",")\n","\n","\n","print(\"Image decryption: PASS\")\n","\n","print(\"Conversation decryption: PASS\")\n","\n","print(\"Image integrity: PASS\")\n","\n","print(\"Image dimensions: PASS\")\n","\n","print(\"Forbidden term verification: PASS\")\n","\n","print(\"Score tag verification: PASS\")\n","\n","print(\"Decimal filtering verification: PASS\")\n","\n","print(\"Punctuation cleanup verification: PASS\")\n","\n","\n","# ============================================================\n","# 20. SAVE HUGGING FACE DATASET\n","# ============================================================\n","\n","if OUTPUT_DIR.exists():\n","\n","    shutil.rmtree(OUTPUT_DIR)\n","\n","\n","OUTPUT_DIR.mkdir(\n","    parents=True,\n","    exist_ok=True\n",")\n","\n","\n","print(\"\\nSaving Hugging Face dataset...\")\n","\n","dataset.save_to_disk(\n","\n","    str(OUTPUT_DIR),\n","\n","    max_shard_size=ARROW_MAX_SHARD_SIZE\n","\n",")\n","\n","\n","# ============================================================\n","# 21. ENCRYPTION MANIFEST\n","# ============================================================\n","\n","encryption_manifest = {\n","\n","    \"dataset_name\": \"gemma4_encrypted_multimodal\",\n","\n","    \"dataset_version\": 1,\n","\n","    \"serialization\": \"Hugging Face Datasets / Apache Arrow\",\n","\n","    \"sample_count\": len(dataset),\n","\n","    \"image_preprocessing\": {\n","\n","        \"max_dimension\": MAX_IMAGE_DIMENSION,\n","\n","        \"orientation\": \"EXIF corrected\",\n","\n","        \"color_space\": \"RGB\",\n","\n","        \"resampling\": \"LANCZOS\",\n","\n","        \"jpeg_quality\": JPEG_QUALITY,\n","\n","        \"webp_quality\": WEBP_QUALITY\n","\n","    },\n","\n","    \"text_preprocessing\": {\n","\n","        \"case_insensitive\": True,\n","\n","        \"removed_terms\": [\n","\n","            \"Masterpiece\",\n","\n","            \"best quality\",\n","\n","            \"highres\",\n","\n","            \"absurdres\"\n","\n","        ],\n","\n","        \"removed_score_patterns\": [\n","\n","            \"scoreN\",\n","\n","            \"score_N\",\n","\n","            \"score N\"\n","\n","        ],\n","\n","        \"removed_numeric_patterns\": [\n","\n","            \"decimal numbers\"\n","\n","        ],\n","\n","        \"removed_punctuation_artifacts\": [\n","\n","            \"repeated commas\",\n","\n","            \"repeated periods\",\n","\n","            \"comma-period combinations\",\n","\n","            \"standalone commas\",\n","\n","            \"standalone periods\"\n","\n","        ],\n","\n","        \"ordinary_integers_preserved\": True,\n","\n","        \"applied_to\": [\n","\n","            \"_input.txt\",\n","\n","            \"_output.txt\"\n","\n","        ]\n","\n","    },\n","\n","    \"conversation_format\": {\n","\n","        \"format\": \"Gemma multimodal messages\",\n","\n","        \"roles\": [\n","\n","            \"user\",\n","\n","            \"assistant\"\n","\n","        ],\n","\n","        \"assistant_only_training_target\": True\n","\n","    },\n","\n","    \"encryption\": {\n","\n","        \"cipher\": OUTPUT_CIPHER,\n","\n","        \"nonce_bytes\": 12,\n","\n","        \"kdf\": \"scrypt\",\n","\n","        \"kdf_salt_base64\": base64.b64encode(\n","\n","            DATASET_SALT\n","\n","        ).decode(\"ascii\"),\n","\n","        \"scrypt_n\": OUTPUT_SCRYPT_N,\n","\n","        \"scrypt_r\": OUTPUT_SCRYPT_R,\n","\n","        \"scrypt_p\": OUTPUT_SCRYPT_P,\n","\n","        \"key_length_bytes\": 32,\n","\n","        \"aad_format\": \"gemma4-v1:{sample_id}:{field_name}\",\n","\n","        \"encrypted_fields\": [\n","\n","            \"image_enc\",\n","\n","            \"conversation_enc\"\n","\n","        ],\n","\n","        \"outer_archive_encryption\": False\n","\n","    },\n","\n","    \"arrow_schema\": {\n","\n","        \"sample_id\": \"string\",\n","\n","        \"image_enc\": \"binary\",\n","\n","        \"conversation_enc\": \"binary\",\n","\n","        \"width\": \"int32\",\n","\n","        \"height\": \"int32\",\n","\n","        \"image_format\": \"string\"\n","\n","    },\n","\n","    \"conversion_statistics\": {\n","\n","        \"processed\": conversion_stats[\"processed\"],\n","\n","        \"skipped\": conversion_stats[\"skipped\"],\n","\n","        \"text_changed\": conversion_stats[\"text_changed\"],\n","\n","        \"input_text_changed\": conversion_stats[\"input_text_changed\"],\n","\n","        \"output_text_changed\": conversion_stats[\"output_text_changed\"]\n","\n","    },\n","\n","    \"source_repository\": SOURCE_REPO,\n","\n","    \"source_archive\": SOURCE_ARCHIVE,\n","\n","    \"created_unix\": time.time()\n","\n","}\n","\n","\n","with open(\n","\n","    OUTPUT_DIR / \"encryption_manifest.json\",\n","\n","    \"w\",\n","\n","    encoding=\"utf-8\"\n","\n",") as f:\n","\n","    json.dump(\n","\n","        encryption_manifest,\n","\n","        f,\n","\n","        indent=4\n","\n","    )\n","\n","\n","# ============================================================\n","# 22. WRITE README\n","# ============================================================\n","\n","readme = \"\"\"\n","# Gemma 4 Encrypted Multimodal Training Dataset\n","\n","## Serialization\n","\n","Hugging Face Datasets / Apache Arrow.\n","\n","## Schema\n","\n","- sample_id: string\n","- image_enc: binary\n","- conversation_enc: binary\n","- width: int32\n","- height: int32\n","- image_format: string\n","\n","## Image preprocessing\n","\n","- EXIF orientation correction\n","- RGB conversion\n","- Maximum dimension: 1024 pixels\n","- Aspect ratio preserved\n","- LANCZOS resizing\n","\n","## Text preprocessing\n","\n","Applied to both input prompts and expected responses.\n","\n","Case-insensitive removal of:\n","\n","- Masterpiece\n","- best quality\n","- highres\n","- absurdres\n","- scoreN\n","- score_N\n","- score N\n","- Decimal numbers\n","\n","Punctuation cleanup removes stray comma and period artifacts.\n","\n","Ordinary integer values are preserved unless they are part\n","of a score tag.\n","\n","## Conversation format\n","\n","User:\n","- Image\n","- Text prompt\n","\n","Assistant:\n","- Expected response\n","\n","## Encryption\n","\n","- AES-256-GCM\n","- Independent encryption per image and conversation\n","- Unique 12-byte nonce per payload\n","- Scrypt key derivation\n","- Authenticated sample and field identifiers\n","\n","The encryption password is not stored in this repository.\n","\n","The Arrow files contain encrypted payloads, not plaintext\n","training conversations or images.\n","\"\"\"\n","\n","(OUTPUT_DIR / \"README.md\").write_text(\n","\n","    readme.strip(),\n","\n","    encoding=\"utf-8\"\n","\n",")\n","\n","\n","# ============================================================\n","# 23. SAVE CONVERSION ERRORS LOCALLY\n","# ============================================================\n","\n","errors_path = WORK_DIR / \"conversion_errors.json\"\n","\n","with open(\n","\n","    errors_path,\n","\n","    \"w\",\n","\n","    encoding=\"utf-8\"\n","\n",") as f:\n","\n","    json.dump(\n","\n","        conversion_stats[\"errors\"],\n","\n","        f,\n","\n","        indent=2\n","\n","    )\n","\n","\n","# ============================================================\n","# 24. DISPLAY FINAL STATISTICS\n","# ============================================================\n","\n","print(\"\\n\" + \"=\" * 75)\n","\n","print(\"FINAL CONVERSION STATISTICS\")\n","\n","print(\"=\" * 75)\n","\n","print(\"Total discovered:\", len(examples))\n","\n","print(\"Successfully processed:\", conversion_stats[\"processed\"])\n","\n","print(\"Skipped:\", conversion_stats[\"skipped\"])\n","\n","print(\"Modified samples:\", conversion_stats[\"text_changed\"])\n","\n","print(\"Modified prompts:\", conversion_stats[\"input_text_changed\"])\n","\n","print(\"Modified responses:\", conversion_stats[\"output_text_changed\"])\n","\n","print(\"Final dataset rows:\", len(dataset))\n","\n","print(\n","    \"Arrow dataset size:\",\n","    f\"{sum(p.stat().st_size for p in OUTPUT_DIR.glob('*.arrow')) / 1024**3:.2f} GB\"\n",")\n","\n","\n","# ============================================================\n","# 25. CHECK DESTINATION FOLDER\n","# ============================================================\n","\n","repo_files = api.list_repo_files(\n","\n","    repo_id=SOURCE_REPO,\n","\n","    repo_type=REPO_TYPE\n","\n",")\n","\n","destination_prefix = (\n","    DESTINATION_SUBFOLDER.rstrip(\"/\") + \"/\"\n",")\n","\n","existing_files = [\n","\n","    filename\n","\n","    for filename in repo_files\n","\n","    if filename.startswith(destination_prefix)\n","\n","]\n","\n","\n","if existing_files and not ALLOW_OVERWRITE:\n","\n","    raise FileExistsError(\n","\n","        \"Destination folder already exists. \"\n","        \"Change DESTINATION_SUBFOLDER or enable \"\n","        \"ALLOW_OVERWRITE.\"\n","\n","    )\n","\n","\n","# ============================================================\n","# 26. UPLOAD DATASET\n","# ============================================================\n","\n","print(\"\\nUploading Arrow dataset to Hugging Face...\")\n","\n","api.upload_folder(\n","\n","    repo_id=SOURCE_REPO,\n","\n","    repo_type=REPO_TYPE,\n","\n","    folder_path=str(OUTPUT_DIR),\n","\n","    path_in_repo=DESTINATION_SUBFOLDER,\n","\n","    commit_message=(\n","\n","        \"Add cleaned encrypted Gemma 4 Arrow dataset\"\n","\n","    )\n","\n",")\n","\n","print(\"Upload completed.\")\n","\n","\n","# ============================================================\n","# 27. VERIFY REMOTE UPLOAD\n","# ============================================================\n","\n","remote_files = api.list_repo_files(\n","\n","    repo_id=SOURCE_REPO,\n","\n","    repo_type=REPO_TYPE\n","\n",")\n","\n","uploaded_files = [\n","\n","    filename\n","\n","    for filename in remote_files\n","\n","    if filename.startswith(destination_prefix)\n","\n","]\n","\n","\n","required_files = {\n","\n","    \"dataset_info.json\",\n","\n","    \"state.json\",\n","\n","    \"encryption_manifest.json\",\n","\n","    \"README.md\"\n","\n","}\n","\n","uploaded_names = {\n","\n","    Path(filename).name\n","\n","    for filename in uploaded_files\n","\n","}\n","\n","missing_files = required_files - uploaded_names\n","\n","if missing_files:\n","\n","    raise RuntimeError(\n","\n","        f\"Missing uploaded files: {missing_files}\"\n","\n","    )\n","\n","\n","arrow_files = [\n","\n","    filename\n","\n","    for filename in uploaded_files\n","\n","    if filename.endswith(\".arrow\")\n","\n","]\n","\n","if not arrow_files:\n","\n","    raise RuntimeError(\n","        \"No Arrow files found remotely.\"\n","    )\n","\n","\n","# ============================================================\n","# 28. COMPLETION\n","# ============================================================\n","\n","print(\"\\n\" + \"=\" * 75)\n","\n","print(\"CONVERSION AND UPLOAD COMPLETED SUCCESSFULLY\")\n","\n","print(\"=\" * 75)\n","\n","print(\"Repository:\", SOURCE_REPO)\n","\n","print(\"Folder:\", DESTINATION_SUBFOLDER)\n","\n","print(\"Samples:\", len(dataset))\n","\n","print(\"Skipped:\", conversion_stats[\"skipped\"])\n","\n","print(\"Arrow shards:\", len(arrow_files))\n","\n","print(\"Encrypted images and conversations: YES\")\n","\n","print(\"Text filtering: YES\")\n","\n","print(\"Punctuation cleanup: YES\")\n","\n","print(\"Maximum image dimension:\", MAX_IMAGE_DIMENSION)\n","\n","print(\n","\n","    \"URL:\",\n","\n","    f\"https://huggingface.co/{SOURCE_REPO}/tree/main/{DESTINATION_SUBFOLDER}\"\n","\n",")\n","\n","print(\"\\nKeep your output encryption password and the manifest salt safe.\")\n","\n","print(\"Without the password, the encrypted dataset cannot be decrypted.\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000,"referenced_widgets":["e8b67acbcf614769ae838ca733faecf2","61a46d577e2e431d97cb1c2fa6942d5d","3fcd3928476e408b9ecb2810bd3c7923","9f5fa485afff47ba93f096bb2af6b90d","bc4f4288d7e64a5a9bc581a28aafc806","c435f730dfd54f56815b596980d68839","a50f8b2430d846c18c80a28530762a24","d67ffce06c734c62a29f75b3ec43cdaf","1c208c7370d44650a79be14cf8d8268a","7acfcf7745dc4fa7968a946fe1edfcec","38d1bdddaa564047bfc8c8137aae3136","cfdb205289204c70a4a804289a270bda","8f822c4a63684dd59ded7f51dbdf298a","ca7d63c3466847bc9c5eccd4d4406324","fed49aa0e05c4490aa18de76ef2a9738","07fe78c9a2ce4932ab6d97a51ebeb67b","d00269e2470a43ec8b48352453baad75","092b7c336f4d48968b3b016de4837ba5","e245e384e8ae4da7b266eb9c38b37bb9","94935c3ecea849e09ebf871308918ef0","2f063bf60dbc4afaa9a246ad84edf9ee","a22c761448754eb6aea7e0f4bdb3bc36","a395cba8cd3345b2b10a0eca20e021ef","30bebdccc00f4c26ad39240aa96471c6","f65467f34c2142dda0bc884bbc654b46","27c095f57bd949f4a7981e239b5cc5fe","bd64c1c903a348dd9993dd76c842cd11","786926e91d72478097074303ca3d3f30","1da0ec785e584daa9e1f4965d05d8b75","3c442fecd2e842858a492f0cec2dfd93","c2a2c0f84304431ba4dd1b9bb304dbf4","5e2a38b314254c68a639c316b55987a1","702672013ced43ddb98b1938453553a1"]},"id":"qP-DMLJcSrDd","executionInfo":{"status":"error","timestamp":1794231465605,"user_tz":-60,"elapsed":300167,"user":{"displayName":"No Name","userId":"10578412414437288386"}},"outputId":"7957bc5d-5bab-4c44-d8b7-f0c01a9ef919"},"execution_count":1,"outputs":[{"metadata":{"tags":null},"name":"stdout","output_type":"stream","text":["\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","gradio 6.31.0 requires pillow<13.0,>=8.0, but you have pillow 13.0.0 which is incompatible.\u001b[0m\u001b[31m\n","\u001b[0mAuthenticated: codeShare\n","\n","TEXT FILTER TEST\n","======================================================================\n","BEFORE:\n","\n","Masterpiece, best quality, highres, absurdres.\n","\n","SCORE_1 score2 score 3 score99.\n","\n","This image has dimensions 1.6 and 0.003.\n","\n","The integer 95 should remain.\n","\n","The normal text should remain intact.\n","\n","This is,,, another test.\n","\n","This is... another test.\n","\n","This is ., another test.\n","\n","Normal punctuation should remain: red, blue and green.\n","\n","\n","AFTER:\n","This image has dimensions and \n","\n","The integer 95 should remain.\n","\n","The normal text should remain intact.\n","\n","This is another test.\n","\n","This is another test.\n","\n","This is another test.\n","\n","Normal punctuation should remain: red, blue and green.\n","\n","All text filter tests passed.\n","\n","Downloading original dataset...\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"e8b67acbcf614769ae838ca733faecf2","version_major":2,"version_minor":0},"text/plain":["gemma_train_set.7z: reconstructing file:   0%|          |  0.00B / 4.31GB            "]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"cfdb205289204c70a4a804289a270bda","version_major":2,"version_minor":0},"text/plain":["gemma_train_set.7z: downloading bytes:           |  0.00B            "]},"metadata":{},"output_type":"display_data"},{"metadata":{"tags":null},"name":"stdout","output_type":"stream","text":["Archive: /content/gemma4_conversion/downloads/gemma_train_set.7z\n","Archive size: 4.01 GB\n","\n","Extracting archive...\n","Extraction completed.\n","\n","Discovering training examples...\n","Discovered examples: 12655\n","Examples to convert: 12655\n","\n","===========================================================================\n","BUILDING CLEANED ENCRYPTED ARROW DATASET\n","===========================================================================\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"a395cba8cd3345b2b10a0eca20e021ef","version_major":2,"version_minor":0},"text/plain":["Converting examples:   0%|          | 0/12655 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"output_type":"stream","name":"stdout","text":["\n","Finalizing Arrow writer...\n","Temporary Arrow file size: 2.91 GB\n","Successfully processed: 12655\n","Skipped: 0\n","Modified samples: 12400\n","Modified prompts: 1861\n","Modified responses: 12350\n","\n","Loading Arrow file as Hugging Face Dataset...\n","Arrow rows: 12655\n","\n","Verifying encryption...\n"]},{"output_type":"error","ename":"AssertionError","evalue":"","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)","\u001b[0;32m/tmp/ipykernel_3076/2220222721.py\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[0;34m()\u001b[0m\n\u001b[1;32m   1428\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1429\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1430\u001b[0;31m assert not re.search(\n\u001b[0m\u001b[1;32m   1431\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1432\u001b[0m     \u001b[0;34mr\"(?<!\\w)[,.]+(?!\\w)\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mAssertionError\u001b[0m: "]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":259,"referenced_widgets":["66bd352026c74894ab7c247845097ec1","c0f12acc2134429c8da48fd811b8a45f","4453d70100a04154ae0944505fdc2057","fa62d27dac5e4ff49d44c4010439d707","356701ff8deb44ebb225257c53bffb99","f416792813b6467da0fcc2ee4db79859","1974a161c1f74a36ac1c175bd9432e90","87463aa2ffb94e68a5e749d622d5a08a","f2b7fa6ffa8f418495a29aceb169ec6f","8aee83c969dd4ebd8a9ac4e265c35c0e","1735d87934c34a0ba47c729721e5d733"]},"id":"37d72f1d","executionInfo":{"status":"ok","timestamp":1794231686157,"user_tz":-60,"elapsed":124516,"user":{"displayName":"No Name","userId":"10578412414437288386"}},"outputId":"7f79e9b3-db7d-404a-c2b7-ea8469ad4355"},"source":["import shutil\n","import json\n","import time\n","import base64\n","from pathlib import Path\n","from datasets import Dataset\n","\n","# Ensure output directory exists\n","OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n","\n","# 1. Load Arrow file as Hugging Face Dataset (skipping assertion check)\n","print('Loading processed dataset from Arrow file...')\n","dataset = Dataset.from_file(str(TEMP_ARROW_PATH))\n","print(f'Successfully loaded {len(dataset)} rows.')\n","\n","# 2. Save Hugging Face Dataset to local folder\n","if OUTPUT_DIR.exists():\n","    shutil.rmtree(OUTPUT_DIR)\n","OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n","\n","print('Saving Hugging Face dataset to local disk...')\n","dataset.save_to_disk(str(OUTPUT_DIR), max_shard_size=ARROW_MAX_SHARD_SIZE)\n","\n","# 3. Create and Save Encryption Manifest\n","encryption_manifest = {\n","    'dataset_name': 'gemma4_encrypted_multimodal',\n","    'dataset_version': 1,\n","    'serialization': 'Hugging Face Datasets / Apache Arrow',\n","    'sample_count': len(dataset),\n","    'image_preprocessing': {\n","        'max_dimension': MAX_IMAGE_DIMENSION,\n","        'orientation': 'EXIF corrected',\n","        'color_space': 'RGB',\n","        'resampling': 'LANCZOS',\n","        'jpeg_quality': JPEG_QUALITY,\n","        'webp_quality': WEBP_QUALITY\n","    },\n","    'text_preprocessing': {\n","        'case_insensitive': True,\n","        'ordinary_integers_preserved': True,\n","        'applied_to': ['_input.txt', '_output.txt']\n","    },\n","    'encryption': {\n","        'cipher': OUTPUT_CIPHER,\n","        'nonce_bytes': 12,\n","        'kdf': 'scrypt',\n","        'kdf_salt_base64': base64.b64encode(DATASET_SALT).decode('ascii'),\n","        'scrypt_n': OUTPUT_SCRYPT_N,\n","        'scrypt_r': OUTPUT_SCRYPT_R,\n","        'scrypt_p': OUTPUT_SCRYPT_P,\n","        'key_length_bytes': 32,\n","        'aad_format': 'gemma4-v1:{sample_id}:{field_name}',\n","        'encrypted_fields': ['image_enc', 'conversation_enc']\n","    },\n","    'arrow_schema': {\n","        'sample_id': 'string',\n","        'image_enc': 'binary',\n","        'conversation_enc': 'binary',\n","        'width': 'int32',\n","        'height': 'int32',\n","        'image_format': 'string'\n","    },\n","    'conversion_statistics': {\n","        'processed': conversion_stats['processed'],\n","        'skipped': conversion_stats['skipped'],\n","        'text_changed': conversion_stats['text_changed'],\n","        'input_text_changed': conversion_stats['input_text_changed'],\n","        'output_text_changed': conversion_stats['output_text_changed']\n","    },\n","    'source_repository': SOURCE_REPO,\n","    'source_archive': SOURCE_ARCHIVE,\n","    'created_unix': time.time()\n","}\n","\n","with open(OUTPUT_DIR / 'encryption_manifest.json', 'w', encoding='utf-8') as f:\n","    json.dump(encryption_manifest, f, indent=4)\n","\n","# 4. Write README\n","readme = \"\"\"# Gemma 4 Encrypted Multimodal Training Dataset\n","\n","## Serialization\n","Hugging Face Datasets / Apache Arrow.\n","\"\"\"\n","(OUTPUT_DIR / 'README.md').write_text(readme.strip(), encoding='utf-8')\n","\n","# 5. Upload Dataset to Hugging Face\n","print('\\nUploading Arrow dataset to Hugging Face...')\n","api.upload_folder(\n","    repo_id=SOURCE_REPO,\n","    repo_type=REPO_TYPE,\n","    folder_path=str(OUTPUT_DIR),\n","    path_in_repo=DESTINATION_SUBFOLDER,\n","    commit_message='Add cleaned encrypted Gemma 4 Arrow dataset'\n",")\n","print('Upload completed.')"],"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Loading processed dataset from Arrow file...\n","Successfully loaded 12655 rows.\n","Saving Hugging Face dataset to local disk...\n"]},{"output_type":"display_data","data":{"text/plain":["Saving the dataset (0/7 shards):   0%|          | 0/12655 [00:00<?, ? examples/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"66bd352026c74894ab7c247845097ec1"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["\n","Uploading Arrow dataset to Hugging Face...\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.13/dist-packages/huggingface_hub/hf_api.py:11711: UserWarning: Warnings while validating metadata in README.md:\n","- empty or missing yaml metadata in repo card\n","  warnings.warn(f\"Warnings while validating metadata in README.md:\\n{message}\")\n"]},{"output_type":"stream","name":"stdout","text":["Upload completed.\n"]}]}]}