{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "gpuType": "T4" }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU", "widgets": { "application/vnd.jupyter.widget-state+json": { "105fb75dea9c468d98e36ba48b91d379": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_0dbda088ec90412e959e6290392d97dc", "IPY_MODEL_46f8cb9a4775444db4a2a7b2e0a72bf0", "IPY_MODEL_851b3bbe139244aabede18801d4a923a" ], "layout": "IPY_MODEL_21039746a8844d88883a517406891d50" } }, "0dbda088ec90412e959e6290392d97dc": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_d3e0703dd70e486fa923ac2c37b1c879", "placeholder": "​", "style": "IPY_MODEL_4f7ad333a1af45f48e174d01b34dc561", "value": "tokenizer_config.json: 100%" } }, "46f8cb9a4775444db4a2a7b2e0a72bf0": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_ac84da5392dd4778b3779d85c182f357", "max": 1289, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_1388398ab4f74ab0831100350a1d99a7", "value": 1289 } }, "851b3bbe139244aabede18801d4a923a": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_37a7646419974760845d692d0ec9ee44", "placeholder": "​", "style": "IPY_MODEL_8dca6c4e6aee4c708b3760ab5d103c28", "value": " 1.29k/1.29k [00:00<00:00, 33.1kB/s]" } }, "21039746a8844d88883a517406891d50": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d3e0703dd70e486fa923ac2c37b1c879": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "4f7ad333a1af45f48e174d01b34dc561": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "ac84da5392dd4778b3779d85c182f357": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "1388398ab4f74ab0831100350a1d99a7": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "37a7646419974760845d692d0ec9ee44": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8dca6c4e6aee4c708b3760ab5d103c28": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "b1678de1646e494b863cf49efa51e4b4": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_251e1602b2a740ee8f7f8889ee2149a6", "IPY_MODEL_c2f46f41d87b4a878dbef56baefebe84", "IPY_MODEL_fc3c9dac87ca4c858654c1b1a50890ac" ], "layout": "IPY_MODEL_d6514f03f917423fbf93a551102c28fe" } }, "251e1602b2a740ee8f7f8889ee2149a6": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_19c073826a644890989d6c453e561b8d", "placeholder": "​", "style": "IPY_MODEL_63e88dbb26044265b6a75bb7112e84c7", "value": "tokenizer.model: 100%" } }, "c2f46f41d87b4a878dbef56baefebe84": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_fd5c322c28e64b99a8c0e877a79e59c2", "max": 499723, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_8f737ff411d040dc8bf4aeb95e8c3cd6", "value": 499723 } }, "fc3c9dac87ca4c858654c1b1a50890ac": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_1c5b20fa7e86463db5b40a3226d28a57", "placeholder": "​", "style": "IPY_MODEL_95568254f93e4bd29c151647bcbc36e9", "value": " 500k/500k [00:00<00:00, 7.28MB/s]" } }, "d6514f03f917423fbf93a551102c28fe": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "19c073826a644890989d6c453e561b8d": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "63e88dbb26044265b6a75bb7112e84c7": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "fd5c322c28e64b99a8c0e877a79e59c2": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8f737ff411d040dc8bf4aeb95e8c3cd6": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "1c5b20fa7e86463db5b40a3226d28a57": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "95568254f93e4bd29c151647bcbc36e9": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "bfeb2fe64096492cb11f6e6548cd4085": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_e327c7dfbb21432fb8637ca693623878", "IPY_MODEL_c29e202a0d57417dab74485f36bf4259", "IPY_MODEL_7a05e30b75fd499998fe2d2e1ba49de3" ], "layout": "IPY_MODEL_17c3b47d4ba24f6fbddf947123b3bfbf" } }, "e327c7dfbb21432fb8637ca693623878": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_909d46c781d64f0f85c6eb6b08a0505c", "placeholder": "​", "style": "IPY_MODEL_f335d3386bf74e928678175451e58904", "value": "tokenizer.json: 100%" } }, "c29e202a0d57417dab74485f36bf4259": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_000e1e07eaf44009a643eebb1bbd4333", "max": 1842767, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_d5c3d3d16d174bf39dbffb2320b7143a", "value": 1842767 } }, "7a05e30b75fd499998fe2d2e1ba49de3": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_78273c7bfe144d439a2a16d30652c6d5", "placeholder": "​", "style": "IPY_MODEL_a8649eb6dcbf48d1812b4b6076a5edd3", "value": " 1.84M/1.84M [00:00<00:00, 2.62MB/s]" } }, "17c3b47d4ba24f6fbddf947123b3bfbf": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "909d46c781d64f0f85c6eb6b08a0505c": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f335d3386bf74e928678175451e58904": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "000e1e07eaf44009a643eebb1bbd4333": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d5c3d3d16d174bf39dbffb2320b7143a": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "78273c7bfe144d439a2a16d30652c6d5": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a8649eb6dcbf48d1812b4b6076a5edd3": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "54c3b3b1aaf540568618094a4dd7c2de": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_fc14cbf3ab3e412a9842f818596ea739", "IPY_MODEL_6374f0370842469b95f9d316416fd71e", "IPY_MODEL_40c8035883f44aecb7e2373861f7752e" ], "layout": "IPY_MODEL_9af36ae1fa954607a5fd9207d52bae10" } }, "fc14cbf3ab3e412a9842f818596ea739": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_55d85509469541e09fdb67c16196c72a", "placeholder": "​", "style": "IPY_MODEL_f4da21a5eeef4b4bb278d9f03ee6061b", "value": "special_tokens_map.json: 100%" } }, "6374f0370842469b95f9d316416fd71e": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_cc4dc428f3734b7685e941fac87ec5ba", "max": 551, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_57844d99675a49658b8e002b61e2fc12", "value": 551 } }, "40c8035883f44aecb7e2373861f7752e": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_794b056f88214be3a9c662785d26d009", "placeholder": "​", "style": "IPY_MODEL_c124d1ce85d54577be6572e2b926354f", "value": " 551/551 [00:00<00:00, 7.81kB/s]" } }, "9af36ae1fa954607a5fd9207d52bae10": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "55d85509469541e09fdb67c16196c72a": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f4da21a5eeef4b4bb278d9f03ee6061b": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "cc4dc428f3734b7685e941fac87ec5ba": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "57844d99675a49658b8e002b61e2fc12": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "794b056f88214be3a9c662785d26d009": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "c124d1ce85d54577be6572e2b926354f": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "b4c10becbd7845a2828ef22975db1e74": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_513868ac37684323b30a05bb236f3c40", "IPY_MODEL_074bc3c411ab4be9b3101780efb62d33", "IPY_MODEL_e085f52e520d431692e03b3d6b9890ed" ], "layout": "IPY_MODEL_27578130937a49379b157b92c52aee9b" } }, "513868ac37684323b30a05bb236f3c40": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_aaaa468f9c174c948a6a74204af94440", "placeholder": "​", "style": "IPY_MODEL_ebdc5604a4394bfa9bd847b583f02940", "value": "config.json: 100%" } }, "074bc3c411ab4be9b3101780efb62d33": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_50e0afcbf70b4edea8d6010587ec3bcc", "max": 608, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_921ba7da88b742c19625c1aa1a32a99e", "value": 608 } }, "e085f52e520d431692e03b3d6b9890ed": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_7a38f868e970400b9a4a8e595cb6f08f", "placeholder": "​", "style": "IPY_MODEL_a4e7ed98efe844a5a447faca24bbf326", "value": " 608/608 [00:00<00:00, 67.3kB/s]" } }, "27578130937a49379b157b92c52aee9b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "aaaa468f9c174c948a6a74204af94440": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ebdc5604a4394bfa9bd847b583f02940": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "50e0afcbf70b4edea8d6010587ec3bcc": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "921ba7da88b742c19625c1aa1a32a99e": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "7a38f868e970400b9a4a8e595cb6f08f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a4e7ed98efe844a5a447faca24bbf326": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "339f3ea0ab1b4594bd38d5e274123c1e": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_d2c6320b1edd41829ada662c18dc5a93", "IPY_MODEL_ce14d381e1d3436caa3a911f1f70a9d9", "IPY_MODEL_fe880a41914943398ee851e1932c5c87" ], "layout": "IPY_MODEL_931943c3c69a45969d829d89e437a937" } }, "d2c6320b1edd41829ada662c18dc5a93": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_5a579c5bb74243f4b4ff0daaa3fdee8a", "placeholder": "​", "style": "IPY_MODEL_3ac155c48bac45bba348ac0853582c24", "value": "model.safetensors: 100%" } }, "ce14d381e1d3436caa3a911f1f70a9d9": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_037d002dff9648cc921edd18e024bf5e", "max": 2200119864, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_c6e4a72224e0486f81763399326c0e1a", "value": 2200119864 } }, "fe880a41914943398ee851e1932c5c87": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_89a9dfadcda34370a8b61ac2374514a8", "placeholder": "​", "style": "IPY_MODEL_7dc638cdd4d64236aea2edd3ef286627", "value": " 2.20G/2.20G [00:18<00:00, 245MB/s]" } }, "931943c3c69a45969d829d89e437a937": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5a579c5bb74243f4b4ff0daaa3fdee8a": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "3ac155c48bac45bba348ac0853582c24": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "037d002dff9648cc921edd18e024bf5e": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "c6e4a72224e0486f81763399326c0e1a": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "89a9dfadcda34370a8b61ac2374514a8": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "7dc638cdd4d64236aea2edd3ef286627": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "7df660e81b844248a6deaed0decaef23": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_c93a8faf318049f68729f2e34adef2b2", "IPY_MODEL_2fe5bf5c529249daaea5517dc00a9c8f", "IPY_MODEL_953fb98fe8c84f6aabba26898a98e664" ], "layout": "IPY_MODEL_172d023d55814b02a96f383ab42ca5e1" } }, "c93a8faf318049f68729f2e34adef2b2": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_21c634a44479472fbeb4d20af25a35c6", "placeholder": "​", "style": "IPY_MODEL_79a9c02dd5bf43cc92019014e1a256fd", "value": "generation_config.json: 100%" } }, "2fe5bf5c529249daaea5517dc00a9c8f": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_49062ef858464c9490cb50c509dc99de", "max": 124, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_5f2ee295c18142a2b398afac5cd4608b", "value": 124 } }, "953fb98fe8c84f6aabba26898a98e664": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_40c36240b27d4cc889f312a2129021aa", "placeholder": "​", "style": "IPY_MODEL_d4d9f8b3fcbb4490abf85183f91b2601", "value": " 124/124 [00:00<00:00, 13.6kB/s]" } }, "172d023d55814b02a96f383ab42ca5e1": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "21c634a44479472fbeb4d20af25a35c6": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "79a9c02dd5bf43cc92019014e1a256fd": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "49062ef858464c9490cb50c509dc99de": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5f2ee295c18142a2b398afac5cd4608b": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "40c36240b27d4cc889f312a2129021aa": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d4d9f8b3fcbb4490abf85183f91b2601": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "cells": [ { "cell_type": "markdown", "source": [ "This code is meant to recreate Fig-2 for submission:\n", "\n", "---\n", "\n", "\"**How Much Context Does Natural Language Actually Require?\n", "An Analysis Using LLMs as Statistical Oracles**\"\n", "\n", "---\n", "which evaluates the Minimal Context Length (**MCL**) required for context/next-token pairs extracted from stories sampled from TinyStories. We use TinyLlama as the reference oracle and run the experiments in a minimal configuration to enable faster runtimes on free Colab instances. Note that actual runtime may vary depending on the number of sampled stories. I used ChatGPT to help draft documentation and improve code readability. For questions about this code or alternative runtime setups used for the other figures in the paper, feel free to contact me by email!\n" ], "metadata": { "id": "cxhyUtYIyxip" } }, { "cell_type": "code", "source": [ "# Initial imports\n", "import torch\n", "import os\n", "from transformers import AutoTokenizer, AutoModelForCausalLM\n", "from datasets import load_dataset, disable_caching\n", "from tqdm import tqdm\n", "import matplotlib.pyplot as plt\n", "import json\n", "import numpy as np\n", "from contextlib import nullcontext\n", "import torch.nn.functional as F\n", "import shutil\n", "import tempfile\n", "import requests\n", "import numpy as np\n", "from scipy.optimize import curve_fit\n", "import matplotlib.pyplot as plt\n", "\n", "# Disable all caching for datasets\n", "disable_caching()\n" ], "metadata": { "id": "tlsrybrQvPQv" }, "execution_count": 1, "outputs": [] }, { "cell_type": "code", "source": [ "\n", "def analyze_prediction_confidence_fast(model, input_ids, conf_thresh=0.2):\n", " \"\"\"\n", " Quickly identify positions in a token sequence where the model confidently predicts the correct next token.\n", "\n", " This function runs a single forward pass over the input, compares the top-1 prediction with the ground truth,\n", " and considers it confident if the probability margin between top-1 and top-2 predictions exceeds `conf_thresh`.\n", " Meant to filter out non-confident tokens for the MCL analysis.\n", "\n", " Args:\n", " model (PreTrainedModel): A causal language model.\n", " input_ids (torch.Tensor): 1D tensor of token IDs (not batched).\n", " conf_thresh (float): Minimum required margin (p1 - p2) between top-1 and top-2 token probabilities\n", " to be considered confident.\n", "\n", " Returns:\n", " tuple:\n", " - correct_conf (List[int]): Indices (1-based) of tokens where the model predicted the correct next token\n", " with high confidence.\n", " - input_ids[0] (torch.Tensor): The original token sequence (on CUDA).\n", " \"\"\"\n", " input_ids = input_ids.unsqueeze(0).cuda()\n", " with torch.no_grad():\n", " logits = model(input_ids=input_ids, use_cache=False).logits[:, :-1, :]\n", " probs = torch.nn.functional.softmax(logits, dim=-1).squeeze(0).cpu()\n", " targets = input_ids[0, 1:].cpu()\n", " top_probs, top_ids = torch.topk(probs, 2, dim=-1)\n", " correct_conf = []\n", " # Iterate through the tokens in the dataset\n", " for i in range(len(targets)):\n", " gold = targets[i].item()\n", " top1, top2 = top_ids[i]\n", " p1, p2 = top_probs[i]\n", " # This is where we check if the context is correctly and confidently predicted by the model\n", " if top1.item() == gold and (p1.item() - p2.item()) > conf_thresh:\n", " correct_conf.append(i + 1)\n", " return correct_conf, input_ids[0]\n", "\n", "\n", "def stratified_sample(indices, seq_len, num_samples=100, bins=10):\n", " \"\"\"\n", " Perform stratified sampling over a sequence by dividing it into bins and sampling uniformly from each bin.\n", " Given that we do a filtering using 'analyze_prediction_confidence_fast', we want to make sure we still sample equallt\n", " across the document.\n", "\n", " This function ensures that sampled indices are distributed across the sequence length by:\n", " - Dividing the sequence into `bins` equal-length intervals,\n", " - Sampling approximately `num_samples // bins` from each bin,\n", " - Filling any remaining quota by sampling from unused indices if needed.\n", "\n", " Args:\n", " indices (array-like): The candidate indices to sample from (e.g., token positions).\n", " seq_len (int): Total sequence length (used to define bin edges).\n", " num_samples (int): Total number of samples to select.\n", " bins (int): Number of bins to stratify across.\n", "\n", " Returns:\n", " List[int]: Sorted list of sampled indices stratified across the sequence.\n", " \"\"\"\n", " indices = np.array(indices)\n", " sampled_indices = []\n", " bin_edges = np.linspace(0, seq_len, bins + 1, dtype=int)\n", " samples_per_bin = num_samples // bins\n", " for i in range(bins):\n", " start, end = bin_edges[i], bin_edges[i + 1]\n", " bin_members = indices[(indices >= start) & (indices < end)]\n", " if len(bin_members) > 0:\n", " num = min(samples_per_bin, len(bin_members))\n", " sampled = np.random.choice(bin_members, size=num, replace=False)\n", " sampled_indices.extend(sampled.tolist())\n", " if len(sampled_indices) < num_samples:\n", " extra_needed = num_samples - len(sampled_indices)\n", " extra_pool = np.setdiff1d(indices, sampled_indices)\n", " if len(extra_pool) >= extra_needed:\n", " extra = np.random.choice(extra_pool, size=extra_needed, replace=False)\n", " sampled_indices.extend(extra.tolist())\n", " return sorted(sampled_indices)\n" ], "metadata": { "id": "oY_G-GhcvVyr" }, "execution_count": 2, "outputs": [] }, { "cell_type": "code", "source": [ "\n", "def find_tokenwise_true_context_lengths(model, tokenizer, input_ids, target_indices, threshold=0.2,\n", " step_size=8, min_len=32, use_masking=True,\n", " batch_sizes={32: 1, 512: 1}):\n", " \"\"\"\n", " Determine the minimum context length required to confidently predict each token in a sequence.\n", "\n", " For each target token index in `target_indices`, this function gradually increases the length of context\n", " available to the model (starting from `min_len` up to `idx`), and checks whether the model's top-1 predicted\n", " token matches the ground truth token with a confidence margin greater than `threshold`.\n", "\n", " Args:\n", " model (PreTrainedModel): A causal language model.\n", " tokenizer (PreTrainedTokenizer): The tokenizer corresponding to the model.\n", " input_ids (torch.Tensor): 1D tensor of tokenized input.\n", " target_indices (list[int]): List of token indices for which to determine context sensitivity.\n", " threshold (float): Minimum margin (difference between top-1 and top-2 token probs) to be considered confident.\n", " step_size (int): Increment size for context window during probing.\n", " min_len (int): Minimum context length to begin checking from.\n", " use_masking (bool): If True, uses attention masks to simulate different context sizes.\n", " If False, pads context to simulate variable-length inputs.\n", " batch_sizes (dict): Mapping from context length to batch size, to control memory usage.\n", "\n", " Returns:\n", " dict: Mapping from token index to a dict with key 'conf_ctx_len' indicating\n", " the minimum context length where the token is confidently predicted.\n", " If no such context length is found, value is None.\n", " \"\"\"\n", "\n", " results = {}\n", " input_ids = input_ids.cuda()\n", " for idx in tqdm(target_indices, desc=\"Checking tokens for context lengths\"):\n", "\n", " max_ctx = idx\n", " if max_ctx < min_len:\n", " continue\n", " # Split the context into appropriate subcontext, incrementing by step_size\n", " candidate_lens = list(range(min_len, max_ctx + 1, step_size))\n", " actual_token = input_ids[idx].item()\n", " conf_ctx_len = None\n", "\n", " def get_batch_size(l):\n", " \"\"\"\n", " Helper fucntion used to control memory usage when batching on smaller memory GPUs.\n", " \"\"\"\n", " for k in sorted(batch_sizes.keys(), reverse=True):\n", " if l >= k:\n", " return batch_sizes[k]\n", " return batch_sizes[min(batch_sizes)]\n", "\n", " with torch.no_grad():\n", "\n", " # Go through different subcontext lengths for one context\n", " for batch_start in range(0, len(candidate_lens), get_batch_size(candidate_lens[-1])):\n", " batch_lens = candidate_lens[batch_start: batch_start + get_batch_size(candidate_lens[-1])]\n", "\n", " # If we prefer to use the correct positional encoding, use attention masking to create the subcontexts\n", " if use_masking:\n", " context = input_ids[:idx].unsqueeze(0).repeat(len(batch_lens), 1)\n", " attention_mask = torch.zeros_like(context)\n", " for i, ctx_len in enumerate(batch_lens):\n", " attention_mask[i, -ctx_len:] = 1\n", " context = context.cuda()\n", " attention_mask = attention_mask.cuda()\n", " # Or we could just pass in the shortened context itself\n", " else:\n", " max_len = max(batch_lens)\n", " padded_contexts = []\n", " for l in batch_lens:\n", " slice = input_ids[idx - l:idx]\n", " pad_len = max_len - slice.size(0)\n", " padded = F.pad(slice, (pad_len, 0), value=tokenizer.pad_token_id)\n", " padded_contexts.append(padded)\n", " context = torch.stack(padded_contexts)\n", " attention_mask = None\n", "\n", " outputs = model(input_ids=context, attention_mask=attention_mask)\n", " logits = outputs.logits[:, -1, :]\n", " probs = torch.nn.functional.softmax(logits, dim=-1).cpu()\n", "\n", " # Go through the subcontext logits in the batch to evaluate MCL\n", " for i, ctx_len in enumerate(batch_lens):\n", " top_probs, top_ids = torch.topk(probs[i], 2)\n", " top1, top2 = top_ids.tolist()\n", " p1, p2 = top_probs.tolist()\n", " margin = p1 - p2\n", " # check if a subcontext matches the requirements of MCL\n", " if conf_ctx_len is None and top1 == actual_token and margin > threshold:\n", " conf_ctx_len = ctx_len\n", "\n", " del outputs, logits, probs\n", " torch.cuda.empty_cache()\n", "\n", " # To save computation, stop when the first correct subcontext is selected\n", " if conf_ctx_len is not None:\n", " break\n", "\n", " results[idx] = {\"conf_ctx_len\": conf_ctx_len}\n", " return results\n", "\n" ], "metadata": { "id": "Ragu4YiGvmak" }, "execution_count": 3, "outputs": [] }, { "cell_type": "code", "source": [ "def plot_confidence_pie(total_confident_tokens, total_tokens, processed_stories, save_path):\n", " \"\"\"\n", " Plot a pie chart showing the proportion of confidently predicted tokens.\n", "\n", " Args:\n", " total_confident_tokens (int): Number of tokens confidently predicted.\n", " total_tokens (int): Total number of predicted tokens.\n", " processed_stories (int): Number of processed stories.\n", " save_path (str): Output path for the plot.\n", " \"\"\"\n", " confident_pct = total_confident_tokens / total_tokens if total_tokens > 0 else 0\n", "\n", " plt.figure(figsize=(6, 6))\n", " plt.pie([confident_pct, 1 - confident_pct],\n", " labels=[f\"Confident ({confident_pct:.1%})\", \"Other\"],\n", " colors=[\"green\", \"lightgray\"], autopct=\"%1.1f%%\")\n", " plt.title(f\"Token Classification After {processed_stories} Stories\")\n", " plt.tight_layout()\n", " plt.savefig(save_path)\n", " plt.close()\n", "\n", "\n", "\n", "def plot_context_length_histogram(conf_lens, save_path):\n", " \"\"\"\n", " Plot histogram of minimum context lengths and overlay a power-law fit.\n", "\n", " Args:\n", " conf_lens (List[int]): List of confident context lengths.\n", " save_path (str): Output path for the plot.\n", " \"\"\"\n", " if not conf_lens:\n", " print(\"⚠️ No confident context lengths to plot.\")\n", " return\n", "\n", " plt.figure(figsize=(8, 5))\n", " bins = list(range(32, max(conf_lens) + 16, 16))\n", " hist_vals, bin_edges = np.histogram(conf_lens, bins=bins)\n", " plt.hist(conf_lens, bins=bins, color=\"green\", edgecolor=\"black\", alpha=0.8)\n", " plt.yscale(\"log\")\n", "\n", " # === Power law fit ===\n", " def power_law(x, a, b):\n", " return a * x ** (-b)\n", "\n", " fit_x = 0.5 * (bin_edges[1:-1] + bin_edges[2:])\n", " fit_y = hist_vals[1:]\n", " valid = fit_y > 0\n", " fit_x = fit_x[valid]\n", " fit_y = fit_y[valid]\n", "\n", " b_val = None\n", " if len(fit_x) >= 2:\n", " log_x = np.log(fit_x)\n", " log_y = np.log(fit_y)\n", " try:\n", " popt, _ = curve_fit(lambda x, a, b: a + b * x, log_x, log_y)\n", " a, b = popt\n", " b_val = b\n", " x_fit = np.linspace(min(fit_x), max(fit_x), 100)\n", " y_fit = np.exp(a) * x_fit ** b\n", " plt.plot(x_fit, y_fit, color='red', linewidth=2, label='Power-law fit')\n", " except Exception as e:\n", " print(f\"⚠️ Fit failed: {e}\")\n", "\n", " # === Annotations ===\n", " pct_below_64 = int(100 * sum(x <= 64 for x in conf_lens) / len(conf_lens) + 1)\n", " plt.text(0.98, 0.95,\n", " rf\"$p(l_i \\leq 64) = {pct_below_64:.1f}\\%$\",\n", " transform=plt.gca().transAxes,\n", " fontsize=13, ha='right', va='top')\n", " if b_val is not None:\n", " plt.text(0.98, 0.8,\n", " rf\"$\\hat{{b}} = {b_val:.2f}$\",\n", " transform=plt.gca().transAxes,\n", " fontsize=12, ha='right', va='top', color='red')\n", "\n", " plt.xlabel(\"Minimum Context Length (MCL)\", fontsize=14)\n", " plt.ylabel(\"Token Count (log scale)\", fontsize=14)\n", " plt.title(\"Confident Tokens: Context Length Distribution\", fontsize=15)\n", " plt.tight_layout()\n", " plt.savefig(save_path)\n", " plt.close()\n" ], "metadata": { "id": "DrfUvjMrxWjU" }, "execution_count": 4, "outputs": [] }, { "cell_type": "code", "source": [ "\n", "def run_context_analysis(model_name=\"tinyllama\", num_stories=1, min_tokens=1024,\n", " conf_thresh=0.2, num_samples_per_doc=100, output_dir=\"./output\"):\n", " \"\"\"\n", " Run context length sensitivity analysis on a TinyStories-style dataset using a causal LM.\n", "\n", " For each story:\n", " - Select confident token positions based on prediction margin.\n", " - Stratify and sample tokens.\n", " - Estimate the minimum context length needed for confident correct prediction.\n", " - Save results and generate a histogram + power-law fit of context lengths.\n", " \"\"\"\n", "\n", " # === Clean HuggingFace cache ===\n", " cache_dirs = [\n", " \"/root/.cache/huggingface\",\n", " \"/content/fresh_hf_cache\",\n", " \"/content/hf_cache\",\n", " os.path.expanduser(\"~/.cache/huggingface\"),\n", " \"/tmp/huggingface_cache\",\n", " \"/tmp/hf_datasets_cache\",\n", " os.path.expanduser(\"~/.cache/huggingface/datasets\"),\n", " ]\n", " for cache_dir in cache_dirs:\n", " if os.path.exists(cache_dir):\n", " try:\n", " shutil.rmtree(cache_dir)\n", " print(f\"Removed cache directory: {cache_dir}\")\n", " except:\n", " pass\n", "\n", " torch.manual_seed(42)\n", "\n", " # === Load model ===\n", " if model_name.lower() in [\"tinyllama\", \"tinyllama/tinyllama-1.1b-chat-v1.0\"]:\n", " model_id = \"TinyLlama/TinyLlama-1.1B-Chat-v1.0\"\n", " quant_context = nullcontext()\n", " model_kwargs = {\"torch_dtype\": torch.float16}\n", " else:\n", " raise ValueError(f\"Unsupported model: {model_name}\")\n", "\n", " with quant_context:\n", " print(f\"Loading tokenizer for {model_id}...\")\n", " tokenizer = AutoTokenizer.from_pretrained(model_id)\n", " tokenizer.pad_token = tokenizer.eos_token\n", " print(f\"Loading model {model_id}...\")\n", " model = AutoModelForCausalLM.from_pretrained(model_id, **model_kwargs).cuda().eval()\n", "\n", " # === Load TinyStories ===\n", " print(\"\\U0001f501 Downloading TinyStories from raw URL...\")\n", " url = \"https://huggingface.co/datasets/roneneldan/TinyStories/resolve/main/TinyStoriesV2-GPT4-train.txt\"\n", " response = requests.get(url)\n", " raw_stories = response.text.strip().split(\"\\n\\n\")\n", " dataset = [{\"text\": story.strip()} for story in raw_stories if story.strip()]\n", " text_key = \"text\"\n", "\n", " # === Initialize outputs ===\n", " total_correct_conf = 0\n", " total_tokens = 0\n", " processed = 0\n", " all_ctx_lens = {}\n", " conf_lens = []\n", "\n", "\n", " # Analyze stories one by one, choose examples which are long enough\n", " print(f\"Starting analysis of {num_stories} stories...\")\n", " for ex in tqdm(dataset, desc=\"Scanning stories\"):\n", " if processed >= num_stories:\n", " break\n", "\n", " text = ex[text_key]\n", " tokens = tokenizer(text, return_tensors=\"pt\", truncation=False)[\"input_ids\"]\n", " if tokens.shape[1] < min_tokens:\n", " print(f\"Skipping story {processed}: only {tokens.shape[1]} tokens (need {min_tokens})\")\n", " continue\n", "\n", " input_ids = tokens[0][:min_tokens].cuda()\n", " correct_conf, input_ids = analyze_prediction_confidence_fast(model, input_ids, conf_thresh)\n", "\n", " valid_targets = [idx for idx in correct_conf if idx >= 32]\n", " target_indices = (\n", " stratified_sample(valid_targets, seq_len=len(input_ids), num_samples=num_samples_per_doc)\n", " if len(valid_targets) > num_samples_per_doc else valid_targets\n", " )\n", "\n", " print(f\"Story {processed}: Found {len(correct_conf)} confident tokens, analyzing {len(target_indices)} samples\")\n", "\n", " ctx_lens = find_tokenwise_true_context_lengths(\n", " model=model,\n", " tokenizer=tokenizer,\n", " input_ids=input_ids,\n", " target_indices=target_indices,\n", " threshold=conf_thresh,\n", " step_size=16,\n", " use_masking=True\n", " )\n", "\n", " for tok in ctx_lens:\n", " ctx_lens[tok][\"conf_ctx_len\"] = ctx_lens[tok].get(\"conf_ctx_len\")\n", " if ctx_lens[tok][\"conf_ctx_len\"] is not None:\n", " conf_lens.append(ctx_lens[tok][\"conf_ctx_len\"])\n", "\n", " all_ctx_lens[f\"story_{processed}\"] = ctx_lens\n", "\n", " os.makedirs(output_dir, exist_ok=True)\n", " with open(os.path.join(output_dir, f\"context_lengths_partial.json\"), \"w\") as f:\n", " json.dump(all_ctx_lens, f, indent=2)\n", "\n", " total_correct_conf += len(correct_conf)\n", " total_tokens += input_ids.shape[0] - 1\n", " processed += 1\n", "\n", " # Plot the percentage of tokens filtered to be ones which the model predicts correctly and confidently\n", " plot_confidence_pie(\n", " total_confident_tokens=total_correct_conf,\n", " total_tokens=total_tokens,\n", " processed_stories=processed,\n", " save_path=os.path.join(output_dir, \"summary_classification_progress.png\")\n", " )\n", "\n", " # Plot the MCL values\n", " if conf_lens:\n", " plot_context_length_histogram(\n", " conf_lens=conf_lens,\n", " save_path=os.path.join(output_dir, \"confident_token_ctx_lengths.png\")\n", " )\n", "\n", " with open(os.path.join(output_dir, \"context_lengths.json\"), \"w\") as f:\n", " json.dump(all_ctx_lens, f, indent=2)\n", "\n", " print(f\"Analysis complete! Processed {processed} stories.\")\n", " print(f\"Results saved to {output_dir}\")\n" ], "metadata": { "id": "U35wKaaNEsKP" }, "execution_count": 5, "outputs": [] }, { "cell_type": "code", "source": [ "\n", "# Run the analysis\n", "run_context_analysis(\n", " model_name=\"tinyllama\",\n", " num_stories=100,\n", " min_tokens=1024,\n", " conf_thresh=0.2,\n", " num_samples_per_doc=100,\n", " output_dir=\"./output\"\n", ")" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000, "referenced_widgets": [ "105fb75dea9c468d98e36ba48b91d379", "0dbda088ec90412e959e6290392d97dc", "46f8cb9a4775444db4a2a7b2e0a72bf0", "851b3bbe139244aabede18801d4a923a", "21039746a8844d88883a517406891d50", "d3e0703dd70e486fa923ac2c37b1c879", "4f7ad333a1af45f48e174d01b34dc561", "ac84da5392dd4778b3779d85c182f357", "1388398ab4f74ab0831100350a1d99a7", "37a7646419974760845d692d0ec9ee44", "8dca6c4e6aee4c708b3760ab5d103c28", "b1678de1646e494b863cf49efa51e4b4", "251e1602b2a740ee8f7f8889ee2149a6", "c2f46f41d87b4a878dbef56baefebe84", "fc3c9dac87ca4c858654c1b1a50890ac", "d6514f03f917423fbf93a551102c28fe", "19c073826a644890989d6c453e561b8d", "63e88dbb26044265b6a75bb7112e84c7", "fd5c322c28e64b99a8c0e877a79e59c2", "8f737ff411d040dc8bf4aeb95e8c3cd6", "1c5b20fa7e86463db5b40a3226d28a57", "95568254f93e4bd29c151647bcbc36e9", "bfeb2fe64096492cb11f6e6548cd4085", "e327c7dfbb21432fb8637ca693623878", "c29e202a0d57417dab74485f36bf4259", "7a05e30b75fd499998fe2d2e1ba49de3", "17c3b47d4ba24f6fbddf947123b3bfbf", "909d46c781d64f0f85c6eb6b08a0505c", "f335d3386bf74e928678175451e58904", "000e1e07eaf44009a643eebb1bbd4333", "d5c3d3d16d174bf39dbffb2320b7143a", "78273c7bfe144d439a2a16d30652c6d5", "a8649eb6dcbf48d1812b4b6076a5edd3", "54c3b3b1aaf540568618094a4dd7c2de", "fc14cbf3ab3e412a9842f818596ea739", "6374f0370842469b95f9d316416fd71e", "40c8035883f44aecb7e2373861f7752e", "9af36ae1fa954607a5fd9207d52bae10", "55d85509469541e09fdb67c16196c72a", "f4da21a5eeef4b4bb278d9f03ee6061b", "cc4dc428f3734b7685e941fac87ec5ba", "57844d99675a49658b8e002b61e2fc12", "794b056f88214be3a9c662785d26d009", "c124d1ce85d54577be6572e2b926354f", "b4c10becbd7845a2828ef22975db1e74", "513868ac37684323b30a05bb236f3c40", "074bc3c411ab4be9b3101780efb62d33", "e085f52e520d431692e03b3d6b9890ed", "27578130937a49379b157b92c52aee9b", "aaaa468f9c174c948a6a74204af94440", "ebdc5604a4394bfa9bd847b583f02940", "50e0afcbf70b4edea8d6010587ec3bcc", "921ba7da88b742c19625c1aa1a32a99e", "7a38f868e970400b9a4a8e595cb6f08f", "a4e7ed98efe844a5a447faca24bbf326", "339f3ea0ab1b4594bd38d5e274123c1e", "d2c6320b1edd41829ada662c18dc5a93", "ce14d381e1d3436caa3a911f1f70a9d9", "fe880a41914943398ee851e1932c5c87", "931943c3c69a45969d829d89e437a937", "5a579c5bb74243f4b4ff0daaa3fdee8a", "3ac155c48bac45bba348ac0853582c24", "037d002dff9648cc921edd18e024bf5e", "c6e4a72224e0486f81763399326c0e1a", "89a9dfadcda34370a8b61ac2374514a8", "7dc638cdd4d64236aea2edd3ef286627", "7df660e81b844248a6deaed0decaef23", "c93a8faf318049f68729f2e34adef2b2", "2fe5bf5c529249daaea5517dc00a9c8f", "953fb98fe8c84f6aabba26898a98e664", "172d023d55814b02a96f383ab42ca5e1", "21c634a44479472fbeb4d20af25a35c6", "79a9c02dd5bf43cc92019014e1a256fd", "49062ef858464c9490cb50c509dc99de", "5f2ee295c18142a2b398afac5cd4608b", "40c36240b27d4cc889f312a2129021aa", "d4d9f8b3fcbb4490abf85183f91b2601" ] }, "id": "_WqoGsL2uATu", "outputId": "e3a587d8-901f-4986-da82-dce4e03f2dae" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Removed cache directory: /root/.cache/huggingface\n", "Loading tokenizer for TinyLlama/TinyLlama-1.1B-Chat-v1.0...\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ "/usr/local/lib/python3.11/dist-packages/huggingface_hub/utils/_auth.py:94: UserWarning: \n", "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", "You will be able to reuse this secret in all of your notebooks.\n", "Please note that authentication is recommended but still optional to access public models or datasets.\n", " warnings.warn(\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "tokenizer_config.json: 0%| | 0.00/1.29k [00:00 2048). Running this sequence through the model will result in indexing errors\n" ] }, { "output_type": "stream", "name": "stdout", "text": [ "Skipping story 3: only 459 tokens (need 1024)\n", "Skipping story 3: only 175 tokens (need 1024)\n", "Story 3: Found 539 confident tokens, analyzing 100 samples\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ "\n", "Checking tokens for context lengths: 0%| | 0/100 [00:00