LLM_fine_tuning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "A100",
      "mount_file_id": "1fClP6K-1qZQb4yUvoQAXLoXawS-8n08G",
      "authorship_tag": "ABX9TyPsrPvnb0kCbC6jgqSUnX59",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a2b69e2ea5ec40d28ff3f1345a36ab0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2bca95245d124528b9a44b9b033aab29",
              "IPY_MODEL_f8e9d734380d4ad1a74542b893177e9f",
              "IPY_MODEL_7f8af80255f14db4a509f057e228a7af"
            ],
            "layout": "IPY_MODEL_8956dbcc0a6e48e8865b32cd242109e5"
          }
        },
        "2bca95245d124528b9a44b9b033aab29": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_74a09f1ebae54d4db83f9180ebe87fb0",
            "placeholder": "​",
            "style": "IPY_MODEL_5056b4bb80b04e9fbb51adf1b6b0c6d2",
            "value": "Generating train split: "
          }
        },
        "f8e9d734380d4ad1a74542b893177e9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fcd10fb7f289414c915cd142f236335e",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_49887c817ec94ed3b4a4d4ce255bde4c",
            "value": 1
          }
        },
        "7f8af80255f14db4a509f057e228a7af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_81a509f03125475080e66092e4771b32",
            "placeholder": "​",
            "style": "IPY_MODEL_c31916300a0e4708bbd33a2e5dae346c",
            "value": " 35000/0 [00:00&lt;00:00, 62556.22 examples/s]"
          }
        },
        "8956dbcc0a6e48e8865b32cd242109e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "74a09f1ebae54d4db83f9180ebe87fb0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5056b4bb80b04e9fbb51adf1b6b0c6d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fcd10fb7f289414c915cd142f236335e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "49887c817ec94ed3b4a4d4ce255bde4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "81a509f03125475080e66092e4771b32": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c31916300a0e4708bbd33a2e5dae346c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b9a9351cb24a416ab8064a9455320acb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a6e721fd14874eb38962daf467de5209",
              "IPY_MODEL_0e0e8ea13dba4a75b72df506ae8814eb",
              "IPY_MODEL_c50468d4301045de9a24e3daca128432"
            ],
            "layout": "IPY_MODEL_a7859685c91640d0954dff865d4bffed"
          }
        },
        "a6e721fd14874eb38962daf467de5209": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_70e5d6b5281647b99db50e25e7a2b062",
            "placeholder": "​",
            "style": "IPY_MODEL_f221a547ec0a4c10aa702b5da41fd42c",
            "value": "Generating validation split: "
          }
        },
        "0e0e8ea13dba4a75b72df506ae8814eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d8da0bd30da2421387452ea2888698f4",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8372b72afde1492da55ce40792050517",
            "value": 1
          }
        },
        "c50468d4301045de9a24e3daca128432": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2064a2d4128f463fb1bd1599ed5dbc40",
            "placeholder": "​",
            "style": "IPY_MODEL_120d465a246748a8a4aece221bfd7527",
            "value": " 5000/0 [00:00&lt;00:00, 57678.08 examples/s]"
          }
        },
        "a7859685c91640d0954dff865d4bffed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "70e5d6b5281647b99db50e25e7a2b062": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f221a547ec0a4c10aa702b5da41fd42c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d8da0bd30da2421387452ea2888698f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "8372b72afde1492da55ce40792050517": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2064a2d4128f463fb1bd1599ed5dbc40": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "120d465a246748a8a4aece221bfd7527": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "79697a6d0e69449aa8d2a6b37aa9489c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fe01cfd0ce4643fca758def8fdc13561",
              "IPY_MODEL_b5b2edd2a37d46cdba2799d19ab1cb42",
              "IPY_MODEL_8437bb7c73924595b076604f5645f0b3"
            ],
            "layout": "IPY_MODEL_b90e2cca7fcd4ecba7b05d601c5fdb48"
          }
        },
        "fe01cfd0ce4643fca758def8fdc13561": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_27031e8f67e04206a0528a15b5cf862a",
            "placeholder": "​",
            "style": "IPY_MODEL_9b0f2b8374824959bc6a7eea635693fe",
            "value": "Generating test split: "
          }
        },
        "b5b2edd2a37d46cdba2799d19ab1cb42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c54f786efe5443cdae795bf0247614b3",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8e14b0edf37745808311f4692f6e0168",
            "value": 1
          }
        },
        "8437bb7c73924595b076604f5645f0b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3934ff34d6924177992f1e4b6581e473",
            "placeholder": "​",
            "style": "IPY_MODEL_8dd08b95d76a4cae8b60989c296c4688",
            "value": " 10000/0 [00:00&lt;00:00, 64316.10 examples/s]"
          }
        },
        "b90e2cca7fcd4ecba7b05d601c5fdb48": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "27031e8f67e04206a0528a15b5cf862a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9b0f2b8374824959bc6a7eea635693fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c54f786efe5443cdae795bf0247614b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "8e14b0edf37745808311f4692f6e0168": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3934ff34d6924177992f1e4b6581e473": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8dd08b95d76a4cae8b60989c296c4688": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4429d4661fe14658be51b5c49c702bd7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d2fbfe6d9e1940ae96199f15d341e52c",
              "IPY_MODEL_c508b4aff0854b02a828b4f517661368",
              "IPY_MODEL_dd5a5819d6eb4d3bab16795731c83d4a"
            ],
            "layout": "IPY_MODEL_bc1d8e9b4d564af8b9daf863fd04c791"
          }
        },
        "d2fbfe6d9e1940ae96199f15d341e52c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1ffe1325f860430285457f1a8f153fb5",
            "placeholder": "​",
            "style": "IPY_MODEL_bf5a0cbf45a14b078933655b9b66aff9",
            "value": "Map: 100%"
          }
        },
        "c508b4aff0854b02a828b4f517661368": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_093939e8ead7491f8670ae9180c27856",
            "max": 35000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1cdbb098ca9549339267693359f76c04",
            "value": 35000
          }
        },
        "dd5a5819d6eb4d3bab16795731c83d4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0333c01053cc4c6c806708717608f50b",
            "placeholder": "​",
            "style": "IPY_MODEL_37025166e38d4841a4bc530f021177d0",
            "value": " 35000/35000 [00:23&lt;00:00, 1537.73 examples/s]"
          }
        },
        "bc1d8e9b4d564af8b9daf863fd04c791": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1ffe1325f860430285457f1a8f153fb5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bf5a0cbf45a14b078933655b9b66aff9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "093939e8ead7491f8670ae9180c27856": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1cdbb098ca9549339267693359f76c04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0333c01053cc4c6c806708717608f50b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "37025166e38d4841a4bc530f021177d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5e33d72befe849ac996c924c482b78fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a9ff0f533ae347cfa3fe274802096290",
              "IPY_MODEL_3aa84e296d2a40e6acc8ea902c85fd99",
              "IPY_MODEL_ba86d94c6ea84b62b60913ae0b8b3f28"
            ],
            "layout": "IPY_MODEL_6d3e0afdb4c84f44887b489869a12cb7"
          }
        },
        "a9ff0f533ae347cfa3fe274802096290": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3ad5b054f983465ba2b7f53dae561f78",
            "placeholder": "​",
            "style": "IPY_MODEL_1deeeb6be7914ded9d2b9144d7a848e6",
            "value": "Map: 100%"
          }
        },
        "3aa84e296d2a40e6acc8ea902c85fd99": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3e247a4274e8499bac4f0cc08a87dfc5",
            "max": 5000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c6cb5019c0bc4cd0954df0be440307f6",
            "value": 5000
          }
        },
        "ba86d94c6ea84b62b60913ae0b8b3f28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_010526d2c2e04ddbaa6d046c54c41b4a",
            "placeholder": "​",
            "style": "IPY_MODEL_3077aec71ff54b59a156bb71c7f279a7",
            "value": " 5000/5000 [00:02&lt;00:00, 2039.84 examples/s]"
          }
        },
        "6d3e0afdb4c84f44887b489869a12cb7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3ad5b054f983465ba2b7f53dae561f78": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1deeeb6be7914ded9d2b9144d7a848e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3e247a4274e8499bac4f0cc08a87dfc5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c6cb5019c0bc4cd0954df0be440307f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "010526d2c2e04ddbaa6d046c54c41b4a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3077aec71ff54b59a156bb71c7f279a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "90bea8a0e50648b89a1b4cd2fdb88121": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_01a30549a7534832bf6964f510b1457d",
              "IPY_MODEL_f814e47a91bc4b9ab20576e3ef9ac1f5",
              "IPY_MODEL_eedabcba09834f51931f1e41c5241120"
            ],
            "layout": "IPY_MODEL_937448cb65a9429d913e8149033d6a71"
          }
        },
        "01a30549a7534832bf6964f510b1457d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_27b37fb2063a4d989ddc3e061586d1e1",
            "placeholder": "​",
            "style": "IPY_MODEL_96693f0e04774a0189ee95e338fd5d84",
            "value": "Map: 100%"
          }
        },
        "f814e47a91bc4b9ab20576e3ef9ac1f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b07926f8b9384169ae1afbb5cf0ebe20",
            "max": 10000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c7fffb13131949cb961550c185cc1a29",
            "value": 10000
          }
        },
        "eedabcba09834f51931f1e41c5241120": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_77955029de7c48728479786e56fb158d",
            "placeholder": "​",
            "style": "IPY_MODEL_7b5381ca80de4bcaa594d93f35dec660",
            "value": " 10000/10000 [00:05&lt;00:00, 1986.59 examples/s]"
          }
        },
        "937448cb65a9429d913e8149033d6a71": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "27b37fb2063a4d989ddc3e061586d1e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "96693f0e04774a0189ee95e338fd5d84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b07926f8b9384169ae1afbb5cf0ebe20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c7fffb13131949cb961550c185cc1a29": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "77955029de7c48728479786e56fb158d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7b5381ca80de4bcaa594d93f35dec660": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3ea88f017ac047ea9372a47dc766504b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_63e8aae0bc974b85bfc2360b30e6ead7",
              "IPY_MODEL_6bf1e8ca9da24e8e8d577cbb04d291e9",
              "IPY_MODEL_4faed22e4bfd44fdbfbc02dcdc086539"
            ],
            "layout": "IPY_MODEL_bba5dc2508d1495b9c4ffa1f9aaadc3a"
          }
        },
        "63e8aae0bc974b85bfc2360b30e6ead7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cce9217145d24865a9fd0bbc910d9631",
            "placeholder": "​",
            "style": "IPY_MODEL_4cce60bc11bf4ce19f51ef77ba1650a9",
            "value": "config.json: 100%"
          }
        },
        "6bf1e8ca9da24e8e8d577cbb04d291e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_881bd2594e774f19b3adfa07c53e0705",
            "max": 693,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1f0f856730d640d4a50bc095573a4355",
            "value": 693
          }
        },
        "4faed22e4bfd44fdbfbc02dcdc086539": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_46a5481df93d4932a0346f3849d899b4",
            "placeholder": "​",
            "style": "IPY_MODEL_5dd2b74ca3b0473387f71e68c971a3bc",
            "value": " 693/693 [00:00&lt;00:00, 45.2kB/s]"
          }
        },
        "bba5dc2508d1495b9c4ffa1f9aaadc3a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cce9217145d24865a9fd0bbc910d9631": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4cce60bc11bf4ce19f51ef77ba1650a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "881bd2594e774f19b3adfa07c53e0705": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1f0f856730d640d4a50bc095573a4355": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "46a5481df93d4932a0346f3849d899b4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5dd2b74ca3b0473387f71e68c971a3bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "81346699a562439b89610aa90639595e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_331bdeaa9a0840608b0b12618d247502",
              "IPY_MODEL_8676d1c7ceb644088e91dae2af76b2d5",
              "IPY_MODEL_b75335b1090f414783190ee9cf19a907"
            ],
            "layout": "IPY_MODEL_503d95b919254355b7170aadf2376e78"
          }
        },
        "331bdeaa9a0840608b0b12618d247502": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_874a57430cae4eac830bce3d1b7a5737",
            "placeholder": "​",
            "style": "IPY_MODEL_f170ee9a97164dd88013ff12aaf0a440",
            "value": "model.safetensors: 100%"
          }
        },
        "8676d1c7ceb644088e91dae2af76b2d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bb4dd7aa7d254fa2912dd9338c9f38c7",
            "max": 1118459525,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bf0f895982a64416a99b6fce4efb9178",
            "value": 1118459525
          }
        },
        "b75335b1090f414783190ee9cf19a907": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ce33e78df33941bdb8928855baea5bf7",
            "placeholder": "​",
            "style": "IPY_MODEL_675f877e86a2478c9d505685d0b5369f",
            "value": " 1.12G/1.12G [00:02&lt;00:00, 440MB/s]"
          }
        },
        "503d95b919254355b7170aadf2376e78": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "874a57430cae4eac830bce3d1b7a5737": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f170ee9a97164dd88013ff12aaf0a440": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bb4dd7aa7d254fa2912dd9338c9f38c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bf0f895982a64416a99b6fce4efb9178": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ce33e78df33941bdb8928855baea5bf7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "675f877e86a2478c9d505685d0b5369f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/XindaLi304/LLM_from_Scratch/blob/main/LLM_fine_tuning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install torch lightning matplotlib pandas torchmetrics watermark transformers datasets -U"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J9id01GVccIs",
        "outputId": "d2029e44-7ca5-4c00-cbf3-77ef3cd462cf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.4.0+cu121)\n",
            "Collecting torch\n",
            "  Downloading torch-2.4.1-cp310-cp310-manylinux1_x86_64.whl.metadata (26 kB)\n",
            "Collecting lightning\n",
            "  Downloading lightning-2.4.0-py3-none-any.whl.metadata (38 kB)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (3.7.1)\n",
            "Collecting matplotlib\n",
            "  Downloading matplotlib-3.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (11 kB)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (2.1.4)\n",
            "Collecting pandas\n",
            "  Downloading pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (19 kB)\n",
            "Collecting torchmetrics\n",
            "  Downloading torchmetrics-1.4.2-py3-none-any.whl.metadata (19 kB)\n",
            "Collecting watermark\n",
            "  Downloading watermark-2.4.3-py2.py3-none-any.whl.metadata (1.4 kB)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.44.2)\n",
            "Collecting datasets\n",
            "  Downloading datasets-3.0.0-py3-none-any.whl.metadata (19 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.16.0)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.12.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.13.2)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2024.6.1)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch)\n",
            "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch)\n",
            "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch)\n",
            "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cudnn-cu12==9.1.0.70 (from torch)\n",
            "  Downloading nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch)\n",
            "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch)\n",
            "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-curand-cu12==10.3.2.106 (from torch)\n",
            "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch)\n",
            "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch)\n",
            "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-nccl-cu12==2.20.5 (from torch)\n",
            "  Downloading nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl.metadata (1.8 kB)\n",
            "Collecting nvidia-nvtx-cu12==12.1.105 (from torch)\n",
            "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\n",
            "Collecting triton==3.0.0 (from torch)\n",
            "  Downloading triton-3.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.3 kB)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.6.68-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Requirement already satisfied: PyYAML<8.0,>=5.4 in /usr/local/lib/python3.10/dist-packages (from lightning) (6.0.2)\n",
            "Collecting lightning-utilities<2.0,>=0.10.0 (from lightning)\n",
            "  Downloading lightning_utilities-0.11.7-py3-none-any.whl.metadata (5.2 kB)\n",
            "Requirement already satisfied: packaging<25.0,>=20.0 in /usr/local/lib/python3.10/dist-packages (from lightning) (24.1)\n",
            "Requirement already satisfied: tqdm<6.0,>=4.57.0 in /usr/local/lib/python3.10/dist-packages (from lightning) (4.66.5)\n",
            "Collecting pytorch-lightning (from lightning)\n",
            "  Downloading pytorch_lightning-2.4.0-py3-none-any.whl.metadata (21 kB)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.3.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (4.53.1)\n",
            "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.4.7)\n",
            "Requirement already satisfied: numpy>=1.23 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.26.4)\n",
            "Requirement already satisfied: pillow>=8 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (9.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (3.1.4)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas) (2024.1)\n",
            "Requirement already satisfied: ipython>=6.0 in /usr/local/lib/python3.10/dist-packages (from watermark) (7.34.0)\n",
            "Requirement already satisfied: importlib-metadata>=1.4 in /usr/local/lib/python3.10/dist-packages (from watermark) (8.5.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from watermark) (71.0.4)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.24.6)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.5.15)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.32.3)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.5)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.1)\n",
            "Collecting pyarrow>=15.0.0 (from datasets)\n",
            "  Downloading pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl.metadata (3.3 kB)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n",
            "  Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n",
            "Collecting xxhash (from datasets)\n",
            "  Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
            "Collecting multiprocess (from datasets)\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl.metadata (7.2 kB)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.10.5)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (2.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (24.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.1.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.11.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: zipp>=3.20 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata>=1.4->watermark) (3.20.1)\n",
            "Collecting jedi>=0.16 (from ipython>=6.0->watermark)\n",
            "  Using cached jedi-0.19.1-py2.py3-none-any.whl.metadata (22 kB)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.10/dist-packages (from ipython>=6.0->watermark) (4.4.2)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/dist-packages (from ipython>=6.0->watermark) (0.7.5)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.10/dist-packages (from ipython>=6.0->watermark) (5.7.1)\n",
            "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from ipython>=6.0->watermark) (3.0.47)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.10/dist-packages (from ipython>=6.0->watermark) (2.16.1)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.10/dist-packages (from ipython>=6.0->watermark) (0.2.0)\n",
            "Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.10/dist-packages (from ipython>=6.0->watermark) (0.1.7)\n",
            "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/dist-packages (from ipython>=6.0->watermark) (4.9.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.8)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.8.30)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.5)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /usr/local/lib/python3.10/dist-packages (from jedi>=0.16->ipython>=6.0->watermark) (0.8.4)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/dist-packages (from pexpect>4.3->ipython>=6.0->watermark) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=6.0->watermark) (0.2.13)\n",
            "Downloading torch-2.4.1-cp310-cp310-manylinux1_x86_64.whl (797.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m797.1/797.1 MB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m105.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m88.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m50.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl (664.8 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m664.8/664.8 MB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m18.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m39.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m18.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m176.2/176.2 MB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading triton-3.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (209.4 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.4/209.4 MB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading lightning-2.4.0-py3-none-any.whl (810 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m811.0/811.0 kB\u001b[0m \u001b[31m50.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading matplotlib-3.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.3/8.3 MB\u001b[0m \u001b[31m115.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.0 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.0/13.0 MB\u001b[0m \u001b[31m87.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading torchmetrics-1.4.2-py3-none-any.whl (869 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m869.2/869.2 kB\u001b[0m \u001b[31m54.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading watermark-2.4.3-py2.py3-none-any.whl (7.6 kB)\n",
            "Downloading datasets-3.0.0-py3-none-any.whl (474 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m474.3/474.3 kB\u001b[0m \u001b[31m40.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m12.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading lightning_utilities-0.11.7-py3-none-any.whl (26 kB)\n",
            "Downloading pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl (39.9 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m39.9/39.9 MB\u001b[0m \u001b[31m16.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m13.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pytorch_lightning-2.4.0-py3-none-any.whl (815 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m815.2/815.2 kB\u001b[0m \u001b[31m54.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m20.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hUsing cached jedi-0.19.1-py2.py3-none-any.whl (1.6 MB)\n",
            "Downloading nvidia_nvjitlink_cu12-12.6.68-py3-none-manylinux2014_x86_64.whl (19.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.7/19.7 MB\u001b[0m \u001b[31m96.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: xxhash, triton, pyarrow, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, lightning-utilities, jedi, dill, pandas, nvidia-cusparse-cu12, nvidia-cudnn-cu12, multiprocess, matplotlib, watermark, nvidia-cusolver-cu12, torch, torchmetrics, datasets, pytorch-lightning, lightning\n",
            "  Attempting uninstall: pyarrow\n",
            "    Found existing installation: pyarrow 14.0.2\n",
            "    Uninstalling pyarrow-14.0.2:\n",
            "      Successfully uninstalled pyarrow-14.0.2\n",
            "  Attempting uninstall: nvidia-nccl-cu12\n",
            "    Found existing installation: nvidia-nccl-cu12 2.22.3\n",
            "    Uninstalling nvidia-nccl-cu12-2.22.3:\n",
            "      Successfully uninstalled nvidia-nccl-cu12-2.22.3\n",
            "  Attempting uninstall: pandas\n",
            "    Found existing installation: pandas 2.1.4\n",
            "    Uninstalling pandas-2.1.4:\n",
            "      Successfully uninstalled pandas-2.1.4\n",
            "  Attempting uninstall: matplotlib\n",
            "    Found existing installation: matplotlib 3.7.1\n",
            "    Uninstalling matplotlib-3.7.1:\n",
            "      Successfully uninstalled matplotlib-3.7.1\n",
            "  Attempting uninstall: torch\n",
            "    Found existing installation: torch 2.4.0+cu121\n",
            "    Uninstalling torch-2.4.0+cu121:\n",
            "      Successfully uninstalled torch-2.4.0+cu121\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "cudf-cu12 24.4.1 requires pandas<2.2.2dev0,>=2.0, but you have pandas 2.2.2 which is incompatible.\n",
            "cudf-cu12 24.4.1 requires pyarrow<15.0.0a0,>=14.0.1, but you have pyarrow 17.0.0 which is incompatible.\n",
            "google-colab 1.0.0 requires pandas==2.1.4, but you have pandas 2.2.2 which is incompatible.\n",
            "ibis-framework 8.0.0 requires pyarrow<16,>=2, but you have pyarrow 17.0.0 which is incompatible.\n",
            "torchaudio 2.4.0+cu121 requires torch==2.4.0, but you have torch 2.4.1 which is incompatible.\n",
            "torchvision 0.19.0+cu121 requires torch==2.4.0, but you have torch 2.4.1 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed datasets-3.0.0 dill-0.3.8 jedi-0.19.1 lightning-2.4.0 lightning-utilities-0.11.7 matplotlib-3.9.2 multiprocess-0.70.16 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-9.1.0.70 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.6.68 nvidia-nvtx-cu12-12.1.105 pandas-2.2.2 pyarrow-17.0.0 pytorch-lightning-2.4.0 torch-2.4.1 torchmetrics-1.4.2 triton-3.0.0 watermark-2.4.3 xxhash-3.5.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "a2b69e2ea5ec40d28ff3f1345a36ab0a",
            "2bca95245d124528b9a44b9b033aab29",
            "f8e9d734380d4ad1a74542b893177e9f",
            "7f8af80255f14db4a509f057e228a7af",
            "8956dbcc0a6e48e8865b32cd242109e5",
            "74a09f1ebae54d4db83f9180ebe87fb0",
            "5056b4bb80b04e9fbb51adf1b6b0c6d2",
            "fcd10fb7f289414c915cd142f236335e",
            "49887c817ec94ed3b4a4d4ce255bde4c",
            "81a509f03125475080e66092e4771b32",
            "c31916300a0e4708bbd33a2e5dae346c",
            "b9a9351cb24a416ab8064a9455320acb",
            "a6e721fd14874eb38962daf467de5209",
            "0e0e8ea13dba4a75b72df506ae8814eb",
            "c50468d4301045de9a24e3daca128432",
            "a7859685c91640d0954dff865d4bffed",
            "70e5d6b5281647b99db50e25e7a2b062",
            "f221a547ec0a4c10aa702b5da41fd42c",
            "d8da0bd30da2421387452ea2888698f4",
            "8372b72afde1492da55ce40792050517",
            "2064a2d4128f463fb1bd1599ed5dbc40",
            "120d465a246748a8a4aece221bfd7527",
            "79697a6d0e69449aa8d2a6b37aa9489c",
            "fe01cfd0ce4643fca758def8fdc13561",
            "b5b2edd2a37d46cdba2799d19ab1cb42",
            "8437bb7c73924595b076604f5645f0b3",
            "b90e2cca7fcd4ecba7b05d601c5fdb48",
            "27031e8f67e04206a0528a15b5cf862a",
            "9b0f2b8374824959bc6a7eea635693fe",
            "c54f786efe5443cdae795bf0247614b3",
            "8e14b0edf37745808311f4692f6e0168",
            "3934ff34d6924177992f1e4b6581e473",
            "8dd08b95d76a4cae8b60989c296c4688",
            "4429d4661fe14658be51b5c49c702bd7",
            "d2fbfe6d9e1940ae96199f15d341e52c",
            "c508b4aff0854b02a828b4f517661368",
            "dd5a5819d6eb4d3bab16795731c83d4a",
            "bc1d8e9b4d564af8b9daf863fd04c791",
            "1ffe1325f860430285457f1a8f153fb5",
            "bf5a0cbf45a14b078933655b9b66aff9",
            "093939e8ead7491f8670ae9180c27856",
            "1cdbb098ca9549339267693359f76c04",
            "0333c01053cc4c6c806708717608f50b",
            "37025166e38d4841a4bc530f021177d0",
            "5e33d72befe849ac996c924c482b78fb",
            "a9ff0f533ae347cfa3fe274802096290",
            "3aa84e296d2a40e6acc8ea902c85fd99",
            "ba86d94c6ea84b62b60913ae0b8b3f28",
            "6d3e0afdb4c84f44887b489869a12cb7",
            "3ad5b054f983465ba2b7f53dae561f78",
            "1deeeb6be7914ded9d2b9144d7a848e6",
            "3e247a4274e8499bac4f0cc08a87dfc5",
            "c6cb5019c0bc4cd0954df0be440307f6",
            "010526d2c2e04ddbaa6d046c54c41b4a",
            "3077aec71ff54b59a156bb71c7f279a7",
            "90bea8a0e50648b89a1b4cd2fdb88121",
            "01a30549a7534832bf6964f510b1457d",
            "f814e47a91bc4b9ab20576e3ef9ac1f5",
            "eedabcba09834f51931f1e41c5241120",
            "937448cb65a9429d913e8149033d6a71",
            "27b37fb2063a4d989ddc3e061586d1e1",
            "96693f0e04774a0189ee95e338fd5d84",
            "b07926f8b9384169ae1afbb5cf0ebe20",
            "c7fffb13131949cb961550c185cc1a29",
            "77955029de7c48728479786e56fb158d",
            "7b5381ca80de4bcaa594d93f35dec660",
            "3ea88f017ac047ea9372a47dc766504b",
            "63e8aae0bc974b85bfc2360b30e6ead7",
            "6bf1e8ca9da24e8e8d577cbb04d291e9",
            "4faed22e4bfd44fdbfbc02dcdc086539",
            "bba5dc2508d1495b9c4ffa1f9aaadc3a",
            "cce9217145d24865a9fd0bbc910d9631",
            "4cce60bc11bf4ce19f51ef77ba1650a9",
            "881bd2594e774f19b3adfa07c53e0705",
            "1f0f856730d640d4a50bc095573a4355",
            "46a5481df93d4932a0346f3849d899b4",
            "5dd2b74ca3b0473387f71e68c971a3bc",
            "81346699a562439b89610aa90639595e",
            "331bdeaa9a0840608b0b12618d247502",
            "8676d1c7ceb644088e91dae2af76b2d5",
            "b75335b1090f414783190ee9cf19a907",
            "503d95b919254355b7170aadf2376e78",
            "874a57430cae4eac830bce3d1b7a5737",
            "f170ee9a97164dd88013ff12aaf0a440",
            "bb4dd7aa7d254fa2912dd9338c9f38c7",
            "bf0f895982a64416a99b6fce4efb9178",
            "ce33e78df33941bdb8928855baea5bf7",
            "675f877e86a2478c9d505685d0b5369f"
          ]
        },
        "id": "sCzZ21_GWSdT",
        "outputId": "46bf05f6-54a4-401d-b805-b4cacb5377ef"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Python implementation: CPython\n",
            "Python version       : 3.10.12\n",
            "IPython version      : 7.34.0\n",
            "\n",
            "torch       : 2.4.1\n",
            "lightning   : 2.4.0\n",
            "transformers: 4.44.2\n",
            "\n",
            "Torch CUDA available? True\n",
            "100% | 80.23 MB | 1.61 MB/s | 49.68 sec elapsed"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 50000/50000 [00:56<00:00, 892.34it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Class distribution:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a2b69e2ea5ec40d28ff3f1345a36ab0a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating validation split: 0 examples [00:00, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b9a9351cb24a416ab8064a9455320acb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating test split: 0 examples [00:00, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "79697a6d0e69449aa8d2a6b37aa9489c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Tokenizer input max length: 1000000000000000019884624838656\n",
            "Tokenizer vocabulary size: 250680\n",
            "Example text: I like LLMs! tokenized into {'input_ids': [44, 3269, 67149, 23099, 4], 'attention_mask': [1, 1, 1, 1, 1]}\n",
            "tokens in text form ['I', 'ĠLike', 'ĠLL', 'Ms', '!']\n",
            "Decoded text: I like LLMs\n",
            "I 44\n",
            "like 29726\n",
            "LLMs None\n",
            "! 4\n",
            "Ġ 210\n",
            "LL 17368\n",
            "Ġlike 3269\n",
            "adding LLM into vocab of tokenizer\n",
            "LLMs 250680\n",
            "current tokenizer size 250681\n",
            "Tokenizing dataset...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/35000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4429d4661fe14658be51b5c49c702bd7"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/5000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5e33d72befe849ac996c924c482b78fb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/10000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "90bea8a0e50648b89a1b4cd2fdb88121"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO: Using 16-bit Automatic Mixed Precision (AMP)\n",
            "INFO:lightning.pytorch.utilities.rank_zero:Using 16-bit Automatic Mixed Precision (AMP)\n",
            "INFO: You are using a CUDA device ('NVIDIA A100-SXM4-40GB') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
            "INFO:lightning.pytorch.utilities.rank_zero:You are using a CUDA device ('NVIDIA A100-SXM4-40GB') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/693 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3ea88f017ac047ea9372a47dc766504b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/1.12G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "81346699a562439b89610aa90639595e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of BloomForSequenceClassification were not initialized from the model checkpoint at bigscience/bloom-560m and are newly initialized: ['score.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/lib/python3.10/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n",
            "  self.pid = os.fork()\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 0001/0001 | Batch 0000/35000 | Loss: 45.6562\n",
            "Epoch: 0001/0001 | Batch 0300/35000 | Loss: 0.4683\n",
            "Epoch: 0001/0001 | Batch 0600/35000 | Loss: 0.0026\n",
            "Epoch: 0001/0001 | Batch 0900/35000 | Loss: 1.6504\n",
            "Epoch: 0001/0001 | Batch 1200/35000 | Loss: 0.2108\n",
            "Epoch: 0001/0001 | Batch 1500/35000 | Loss: 0.0461\n",
            "Epoch: 0001/0001 | Batch 1800/35000 | Loss: 0.1351\n",
            "Epoch: 0001/0001 | Batch 2100/35000 | Loss: 0.5659\n",
            "Epoch: 0001/0001 | Batch 2400/35000 | Loss: 0.0000\n",
            "Epoch: 0001/0001 | Batch 2700/35000 | Loss: 0.0516\n",
            "Epoch: 0001/0001 | Batch 3000/35000 | Loss: 0.2139\n",
            "Epoch: 0001/0001 | Batch 3300/35000 | Loss: 0.6309\n",
            "Epoch: 0001/0001 | Batch 3600/35000 | Loss: 0.0555\n",
            "Epoch: 0001/0001 | Batch 3900/35000 | Loss: 0.0036\n",
            "Epoch: 0001/0001 | Batch 4200/35000 | Loss: 0.1830\n",
            "Epoch: 0001/0001 | Batch 4500/35000 | Loss: 0.0015\n",
            "Epoch: 0001/0001 | Batch 4800/35000 | Loss: 0.0784\n",
            "Epoch: 0001/0001 | Batch 5100/35000 | Loss: 1.6660\n",
            "Epoch: 0001/0001 | Batch 5400/35000 | Loss: 1.6445\n",
            "Epoch: 0001/0001 | Batch 5700/35000 | Loss: 0.3235\n",
            "Epoch: 0001/0001 | Batch 6000/35000 | Loss: 0.8672\n",
            "Epoch: 0001/0001 | Batch 6300/35000 | Loss: 0.0492\n",
            "Epoch: 0001/0001 | Batch 6600/35000 | Loss: 0.0054\n",
            "Epoch: 0001/0001 | Batch 6900/35000 | Loss: 0.0000\n",
            "Epoch: 0001/0001 | Batch 7200/35000 | Loss: 1.0352\n",
            "Epoch: 0001/0001 | Batch 7500/35000 | Loss: 1.9199\n",
            "Epoch: 0001/0001 | Batch 7800/35000 | Loss: 0.0081\n",
            "Epoch: 0001/0001 | Batch 8100/35000 | Loss: 0.0000\n",
            "Epoch: 0001/0001 | Batch 8400/35000 | Loss: 0.9370\n",
            "Epoch: 0001/0001 | Batch 8700/35000 | Loss: 6.2617\n",
            "Epoch: 0001/0001 | Batch 9000/35000 | Loss: 0.1466\n",
            "Epoch: 0001/0001 | Batch 9300/35000 | Loss: 0.0655\n",
            "Epoch: 0001/0001 | Batch 9600/35000 | Loss: 0.0073\n",
            "Epoch: 0001/0001 | Batch 9900/35000 | Loss: 0.1040\n",
            "Epoch: 0001/0001 | Batch 10200/35000 | Loss: 0.7891\n",
            "Epoch: 0001/0001 | Batch 10500/35000 | Loss: 0.8926\n",
            "Epoch: 0001/0001 | Batch 10800/35000 | Loss: 0.0819\n",
            "Epoch: 0001/0001 | Batch 11100/35000 | Loss: 0.0071\n",
            "Epoch: 0001/0001 | Batch 11400/35000 | Loss: 0.0002\n",
            "Epoch: 0001/0001 | Batch 11700/35000 | Loss: 0.8979\n",
            "Epoch: 0001/0001 | Batch 12000/35000 | Loss: 0.0144\n",
            "Epoch: 0001/0001 | Batch 12300/35000 | Loss: 0.1071\n",
            "Epoch: 0001/0001 | Batch 12600/35000 | Loss: 0.0055\n",
            "Epoch: 0001/0001 | Batch 12900/35000 | Loss: 0.0011\n",
            "Epoch: 0001/0001 | Batch 13200/35000 | Loss: 0.0287\n",
            "Epoch: 0001/0001 | Batch 13500/35000 | Loss: 0.0054\n",
            "Epoch: 0001/0001 | Batch 13800/35000 | Loss: 0.0005\n",
            "Epoch: 0001/0001 | Batch 14100/35000 | Loss: 0.0623\n",
            "Epoch: 0001/0001 | Batch 14400/35000 | Loss: 0.0172\n",
            "Epoch: 0001/0001 | Batch 14700/35000 | Loss: 0.7437\n",
            "Epoch: 0001/0001 | Batch 15000/35000 | Loss: 0.0008\n",
            "Epoch: 0001/0001 | Batch 15300/35000 | Loss: 0.0020\n",
            "Epoch: 0001/0001 | Batch 15600/35000 | Loss: 0.0754\n",
            "Epoch: 0001/0001 | Batch 15900/35000 | Loss: 1.5967\n",
            "Epoch: 0001/0001 | Batch 16200/35000 | Loss: 0.0002\n",
            "Epoch: 0001/0001 | Batch 16500/35000 | Loss: 0.0013\n",
            "Epoch: 0001/0001 | Batch 16800/35000 | Loss: 0.0120\n",
            "Epoch: 0001/0001 | Batch 17100/35000 | Loss: 0.9746\n",
            "Epoch: 0001/0001 | Batch 17400/35000 | Loss: 0.0257\n",
            "Epoch: 0001/0001 | Batch 17700/35000 | Loss: 0.5044\n",
            "Epoch: 0001/0001 | Batch 18000/35000 | Loss: 0.0589\n",
            "Epoch: 0001/0001 | Batch 18300/35000 | Loss: 0.0116\n",
            "Epoch: 0001/0001 | Batch 18600/35000 | Loss: 0.0049\n",
            "Epoch: 0001/0001 | Batch 18900/35000 | Loss: 1.2578\n",
            "Epoch: 0001/0001 | Batch 19200/35000 | Loss: 0.5928\n",
            "Epoch: 0001/0001 | Batch 19500/35000 | Loss: 1.1074\n",
            "Epoch: 0001/0001 | Batch 19800/35000 | Loss: 0.0022\n",
            "Epoch: 0001/0001 | Batch 20100/35000 | Loss: 1.1533\n",
            "Epoch: 0001/0001 | Batch 20400/35000 | Loss: 0.0439\n",
            "Epoch: 0001/0001 | Batch 20700/35000 | Loss: 3.4141\n",
            "Epoch: 0001/0001 | Batch 21000/35000 | Loss: 2.0762\n",
            "Epoch: 0001/0001 | Batch 21300/35000 | Loss: 0.0021\n",
            "Epoch: 0001/0001 | Batch 21600/35000 | Loss: 0.0790\n",
            "Epoch: 0001/0001 | Batch 21900/35000 | Loss: 0.0798\n",
            "Epoch: 0001/0001 | Batch 22200/35000 | Loss: 0.0610\n",
            "Epoch: 0001/0001 | Batch 22500/35000 | Loss: 0.0001\n",
            "Epoch: 0001/0001 | Batch 22800/35000 | Loss: 0.3696\n",
            "Epoch: 0001/0001 | Batch 23100/35000 | Loss: 0.0002\n",
            "Epoch: 0001/0001 | Batch 23400/35000 | Loss: 0.0158\n",
            "Epoch: 0001/0001 | Batch 23700/35000 | Loss: 0.1179\n",
            "Epoch: 0001/0001 | Batch 24000/35000 | Loss: 0.0048\n",
            "Epoch: 0001/0001 | Batch 24300/35000 | Loss: 0.0228\n",
            "Epoch: 0001/0001 | Batch 24600/35000 | Loss: 0.1159\n",
            "Epoch: 0001/0001 | Batch 24900/35000 | Loss: 0.0169\n",
            "Epoch: 0001/0001 | Batch 25200/35000 | Loss: 0.0002\n",
            "Epoch: 0001/0001 | Batch 25500/35000 | Loss: 1.8965\n",
            "Epoch: 0001/0001 | Batch 25800/35000 | Loss: 0.7627\n",
            "Epoch: 0001/0001 | Batch 26100/35000 | Loss: 1.5176\n",
            "Epoch: 0001/0001 | Batch 26400/35000 | Loss: 0.0074\n",
            "Epoch: 0001/0001 | Batch 26700/35000 | Loss: 0.0121\n",
            "Epoch: 0001/0001 | Batch 27000/35000 | Loss: 0.0420\n",
            "Epoch: 0001/0001 | Batch 27300/35000 | Loss: 0.2042\n",
            "Epoch: 0001/0001 | Batch 27600/35000 | Loss: 0.0012\n",
            "Epoch: 0001/0001 | Batch 27900/35000 | Loss: 0.3440\n",
            "Epoch: 0001/0001 | Batch 28200/35000 | Loss: 0.8105\n",
            "Epoch: 0001/0001 | Batch 28500/35000 | Loss: 0.0372\n",
            "Epoch: 0001/0001 | Batch 28800/35000 | Loss: 0.0029\n",
            "Epoch: 0001/0001 | Batch 29100/35000 | Loss: 0.0045\n",
            "Epoch: 0001/0001 | Batch 29400/35000 | Loss: 0.0265\n",
            "Epoch: 0001/0001 | Batch 29700/35000 | Loss: 0.1649\n",
            "Epoch: 0001/0001 | Batch 30000/35000 | Loss: 0.0149\n",
            "Epoch: 0001/0001 | Batch 30300/35000 | Loss: 0.0001\n",
            "Epoch: 0001/0001 | Batch 30600/35000 | Loss: 0.0013\n",
            "Epoch: 0001/0001 | Batch 30900/35000 | Loss: 6.0469\n",
            "Epoch: 0001/0001 | Batch 31200/35000 | Loss: 0.0515\n",
            "Epoch: 0001/0001 | Batch 31500/35000 | Loss: 0.0559\n",
            "Epoch: 0001/0001 | Batch 31800/35000 | Loss: 0.0105\n",
            "Epoch: 0001/0001 | Batch 32100/35000 | Loss: 0.1019\n",
            "Epoch: 0001/0001 | Batch 32400/35000 | Loss: 0.0023\n",
            "Epoch: 0001/0001 | Batch 32700/35000 | Loss: 0.0001\n",
            "Epoch: 0001/0001 | Batch 33000/35000 | Loss: 0.0037\n",
            "Epoch: 0001/0001 | Batch 33300/35000 | Loss: 0.7847\n",
            "Epoch: 0001/0001 | Batch 33600/35000 | Loss: 0.4363\n",
            "Epoch: 0001/0001 | Batch 33900/35000 | Loss: 0.0230\n",
            "Epoch: 0001/0001 | Batch 34200/35000 | Loss: 0.0111\n",
            "Epoch: 0001/0001 | Batch 34500/35000 | Loss: 0.0132\n",
            "Epoch: 0001/0001 | Batch 34800/35000 | Loss: 0.0559\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Using `past_key_values` as a tuple is deprecated and will be removed in v4.45. Please use an appropriate `Cache` class (https://huggingface.co/docs/transformers/v4.41.3/en/internal/generation_utils#transformers.Cache)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 0001/0001 | Train acc.: 79.44% | Val acc.: 88.46%\n",
            "Time elapsed 78.69 min\n",
            "Test accuracy 88.31%\n"
          ]
        }
      ],
      "source": [
        "# pip install torch lightning matplotlib pandas torchmetrics watermark transformers datasets -U\n",
        "\n",
        "import os\n",
        "import os.path as op\n",
        "import time\n",
        "\n",
        "from datasets import load_dataset\n",
        "from lightning import Fabric\n",
        "import torch\n",
        "from torch.utils.data import DataLoader\n",
        "import torchmetrics\n",
        "from transformers import AutoTokenizer\n",
        "from transformers import AutoModelForSequenceClassification\n",
        "\n",
        "from transformers import BloomTokenizerFast, BloomForCausalLM#use another lib beacuse autotokenizer not support train\n",
        "\n",
        "from watermark import watermark\n",
        "\n",
        "from tokenizers import Tokenizer\n",
        "from tokenizers.models import BPE\n",
        "from tokenizers.trainers import BpeTrainer\n",
        "from tokenizers.pre_tokenizers import Whitespace\n",
        "\n",
        "from local_dataset_utilities import download_dataset, load_dataset_into_to_dataframe, partition_dataset\n",
        "from local_dataset_utilities import IMDBDataset\n",
        "\n",
        "\n",
        "def tokenize_text(batch):\n",
        "    return tokenizer(batch[\"text\"], truncation=True, padding=True, max_length=1024)\n",
        "\n",
        "\n",
        "def train(num_epochs, model, optimizer, train_loader, val_loader, fabric,accumulation_step):\n",
        "\n",
        "    for epoch in range(num_epochs):\n",
        "        train_acc = torchmetrics.Accuracy(\n",
        "            task=\"multiclass\", num_classes=2).to(fabric.device)\n",
        "\n",
        "        for batch_idx, batch in enumerate(train_loader):\n",
        "            model.train()\n",
        "\n",
        "            ### FORWARD AND BACK PROP\n",
        "            outputs = model(\n",
        "                batch[\"input_ids\"],\n",
        "                attention_mask=batch[\"attention_mask\"],\n",
        "                labels=batch[\"label\"]\n",
        "            )\n",
        "            outputs[\"loss\"]=outputs[\"loss\"]/accumulation_step\n",
        "            fabric.backward(outputs[\"loss\"])\n",
        "\n",
        "            ### UPDATE MODEL PARAMETERS\n",
        "            if batch_idx % accumulation_step == 0:\n",
        "              optimizer.step()\n",
        "              optimizer.zero_grad()\n",
        "\n",
        "            ### LOGGING\n",
        "            if not batch_idx % 300:\n",
        "                print(f\"Epoch: {epoch+1:04d}/{num_epochs:04d} \"\n",
        "                      f\"| Batch {batch_idx:04d}/{len(train_loader):04d} \"\n",
        "                      f\"| Loss: {outputs['loss']:.4f}\")\n",
        "\n",
        "            model.eval()\n",
        "            with torch.no_grad():\n",
        "                predicted_labels = torch.argmax(outputs[\"logits\"], 1)\n",
        "                train_acc.update(predicted_labels, batch[\"label\"])\n",
        "\n",
        "        ### MORE LOGGING\n",
        "        model.eval()\n",
        "        with torch.no_grad():\n",
        "            val_acc = torchmetrics.Accuracy(task=\"multiclass\", num_classes=2).to(fabric.device)\n",
        "            for batch in val_loader:\n",
        "                outputs = model(\n",
        "                    batch[\"input_ids\"],\n",
        "                    attention_mask=batch[\"attention_mask\"],\n",
        "                    labels=batch[\"label\"]\n",
        "                )\n",
        "                predicted_labels = torch.argmax(outputs[\"logits\"], 1)\n",
        "                val_acc.update(predicted_labels, batch[\"label\"])\n",
        "\n",
        "            print(f\"Epoch: {epoch+1:04d}/{num_epochs:04d} \"\n",
        "                  f\"| Train acc.: {train_acc.compute()*100:.2f}% \"\n",
        "                  f\"| Val acc.: {val_acc.compute()*100:.2f}%\"\n",
        "                  )\n",
        "            train_acc.reset(), val_acc.reset()\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "\n",
        "    print(watermark(packages=\"torch,lightning,transformers\", python=True))\n",
        "    print(\"Torch CUDA available?\", torch.cuda.is_available())\n",
        "    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "\n",
        "    torch.manual_seed(123)\n",
        "    # torch.use_deterministic_algorithms(True)\n",
        "\n",
        "    ##########################\n",
        "    ### 1 Loading the Dataset:Large Movie Review Dataset\n",
        "    ##########################\n",
        "\n",
        "    download_dataset()\n",
        "    df = load_dataset_into_to_dataframe()\n",
        "    if not (op.exists(\"train.csv\") and op.exists(\"val.csv\") and op.exists(\"test.csv\")):\n",
        "        partition_dataset(df)\n",
        "\n",
        "    imdb_dataset = load_dataset(\n",
        "        \"csv\",\n",
        "        data_files={\n",
        "            \"train\": \"train.csv\",\n",
        "            \"validation\": \"val.csv\",\n",
        "            \"test\": \"test.csv\",\n",
        "        },\n",
        "    )\n",
        "\n",
        "    #########################################\n",
        "    ### 2 Tokenization and Numericalization\n",
        "    #########################################\n",
        "    # utilizing bloom-560m as llm for latter fine tuing and inferring\n",
        "    #tokenizer is bound with certain llm\n",
        "    tokenizer = BloomTokenizerFast.from_pretrained(\"bigscience/bloom-560m\", max_length=1024)\n",
        "    print(\"Tokenizer input max length:\", tokenizer.model_max_length, flush=True)\n",
        "    print(\"Tokenizer vocabulary size:\", tokenizer.vocab_size, flush=True)\n",
        "    print(\"Example text: I like LLMs!\",\"tokenized into\",tokenizer(\"I like LLMs!\"), flush=True)\n",
        "    tokens=tokenizer(\"I like LLMs\")\n",
        "    print(\"tokens in text form\",tokenizer.tokenize(\"I Like LLMs!\"))\n",
        "    #note that its tokenized into 'I', 'ĠLike', 'ĠLL', 'Ms', '!', where Ġ represents blank space\n",
        "    #also note that this tokenizer adopts byte pair encoding(alternative to wordpiece encoding)which break out of vacabulary word into common words like llm to ll and m.\n",
        "    #to have \"LLMs\" wholy tokenized, we need to add llm into tokenizer's vocabulary.\n",
        "    decoded_text=tokenizer.decode(tokens[\"input_ids\"])\n",
        "    print(\"Decoded text:\",decoded_text, flush=True)\n",
        "\n",
        "    #vocabulary inspect\n",
        "    vocab=tokenizer.get_vocab()\n",
        "    print(\"I\",vocab.get(\"I\"))\n",
        "    print(\"like\",vocab.get(\"like\"))\n",
        "    print(\"LLMs\",vocab.get(\"LLMs\"))\n",
        "    print(\"!\",vocab.get(\"!\"))\n",
        "    print(\"Ġ\",vocab.get(\"Ġ\"))\n",
        "    print(\"LL\",vocab.get(\"LL\"))\n",
        "    print(\"Ġlike\",vocab.get(\"Ġlike\"))\n",
        "    # to update tokenizer, we need to add new token into vocab and resize corresponding model's embedding size\n",
        "    #api approach:we can use customized text file to train a tokenizer with BPE/wordpiece encoding Trainer.\n",
        "    print(\"adding LLM into vocab of tokenizer\" )\n",
        "\n",
        "    tokenizer.add_tokens(['LLMs'])\n",
        "    vocab=tokenizer.get_vocab()\n",
        "    print(\"LLMs\",vocab.get(\"LLMs\"))\n",
        "    print(\"current tokenizer size\",len(tokenizer))\n",
        "\n",
        "\n",
        "    print(\"Tokenizing dataset...\", flush=True)\n",
        "    imdb_tokenized = imdb_dataset.map(tokenize_text, batched=True, batch_size=None)\n",
        "    del imdb_dataset\n",
        "    imdb_tokenized.set_format(\"torch\", columns=[\"input_ids\", \"attention_mask\", \"label\"])\n",
        "    os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
        "\n",
        "    #########################################\n",
        "    ### 3 Set Up DataLoaders\n",
        "    #########################################\n",
        "\n",
        "    train_dataset = IMDBDataset(imdb_tokenized, partition_key=\"train\")\n",
        "    val_dataset = IMDBDataset(imdb_tokenized, partition_key=\"validation\")\n",
        "    test_dataset = IMDBDataset(imdb_tokenized, partition_key=\"test\")\n",
        "\n",
        "    train_loader = DataLoader(\n",
        "        dataset=train_dataset,\n",
        "        batch_size=30,\n",
        "        shuffle=True,\n",
        "        num_workers=4,\n",
        "        drop_last=True,\n",
        "    )\n",
        "\n",
        "    val_loader = DataLoader(\n",
        "        dataset=val_dataset,\n",
        "        batch_size=30,\n",
        "        num_workers=4,\n",
        "        drop_last=True,\n",
        "    )\n",
        "\n",
        "    test_loader = DataLoader(\n",
        "        dataset=test_dataset,\n",
        "        batch_size=30,\n",
        "        num_workers=2,\n",
        "        drop_last=True,\n",
        "    )\n",
        "\n",
        "    #########################################\n",
        "    ### 4 Initializing the Model\n",
        "    #########################################\n",
        "\n",
        "    fabric = Fabric(accelerator=\"cuda\", devices=1, precision=\"16-mixed\")\n",
        "    fabric.launch()\n",
        "\n",
        "    model = AutoModelForSequenceClassification.from_pretrained(\n",
        "        \"bigscience/bloom-560m\", num_labels=2)\n",
        "    model.resize_token_embeddings(len(tokenizer))\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=5e-5)\n",
        "\n",
        "    model, optimizer = fabric.setup(model, optimizer)\n",
        "\n",
        "    model=model.compile\n",
        "    train_loader, val_loader, test_loader = fabric.setup_dataloaders(\n",
        "        train_loader, val_loader, test_loader)\n",
        "\n",
        "    #########################################\n",
        "    ### 5 Finetuning\n",
        "    #########################################\n",
        "\n",
        "    start = time.time()\n",
        "    train(\n",
        "        num_epochs=1,\n",
        "        model=model,\n",
        "        optimizer=optimizer,\n",
        "        train_loader=train_loader,\n",
        "        val_loader=val_loader,\n",
        "        fabric=fabric,\n",
        "        accumulation_step=10\n",
        "    )\n",
        "\n",
        "    end = time.time()\n",
        "    elapsed = end-start\n",
        "    print(f\"Time elapsed {elapsed/60:.2f} min\")\n",
        "\n",
        "    with torch.no_grad():\n",
        "        model.eval()\n",
        "        test_acc = torchmetrics.Accuracy(task=\"multiclass\", num_classes=2).to(fabric.device)\n",
        "        for batch in test_loader:\n",
        "            outputs = model(\n",
        "                batch[\"input_ids\"],\n",
        "                attention_mask=batch[\"attention_mask\"],\n",
        "                labels=batch[\"label\"]\n",
        "            )\n",
        "            predicted_labels = torch.argmax(outputs[\"logits\"], 1)\n",
        "            test_acc.update(predicted_labels, batch[\"label\"])\n",
        "\n",
        "    print(f\"Test accuracy {test_acc.compute()*100:.2f}%\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# finetuing a LLM\n",
        "\n",
        " 1.   unknow tokens\n",
        "*   update pretrained tokenizer\n",
        "*   update corresponding pretained model's embedding size as len(current tokenizer)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "\n",
        "2.   new text dataset\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "---\n",
        "\n",
        "3.  fine tuning model\n",
        "*   mixed-percision\n",
        "*   distributed:\n",
        "    tensor-shadering\n",
        "\n",
        "\n",
        "*   maximize use of single gpu:\n",
        "    gradient-accumulation: avoid unstable training caused by small batch due to limited gpu, which is\n",
        "    i.e. use fabric.backward to accumulate scaled gradients through multiple steps, only update weights when steps reach multiple step with optimizer\n",
        "\n",
        "\n",
        "*   following instruct with human feedback:\\\n",
        "f\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "DpOtphYutZMr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline\n",
        "model.eval()\n",
        "label_mapping = {0: \"NEGATIVE\", 1: \"POSITIVE\"}\n",
        "sentiment_analysis = pipeline(\"sentiment-analysis\", model=model, tokenizer=tokenizer,device=\"cuda\")\n",
        "\n",
        "# Example usage\n",
        "user_input = \"this movie is about LLMs\"\n",
        "sentiment = sentiment_analysis(user_input)\n",
        "print(sentiment)  # Outputs: [{'label': 'POSITIVE', 'score': 0.99}]\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UT4jMLNOFxAz",
        "outputId": "eb6b17c7-b884-4d18-c2ed-72d19a8cd925"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The model '_FabricModule' is not supported for sentiment-analysis. Supported models are ['AlbertForSequenceClassification', 'BartForSequenceClassification', 'BertForSequenceClassification', 'BigBirdForSequenceClassification', 'BigBirdPegasusForSequenceClassification', 'BioGptForSequenceClassification', 'BloomForSequenceClassification', 'CamembertForSequenceClassification', 'CanineForSequenceClassification', 'LlamaForSequenceClassification', 'ConvBertForSequenceClassification', 'CTRLForSequenceClassification', 'Data2VecTextForSequenceClassification', 'DebertaForSequenceClassification', 'DebertaV2ForSequenceClassification', 'DistilBertForSequenceClassification', 'ElectraForSequenceClassification', 'ErnieForSequenceClassification', 'ErnieMForSequenceClassification', 'EsmForSequenceClassification', 'FalconForSequenceClassification', 'FlaubertForSequenceClassification', 'FNetForSequenceClassification', 'FunnelForSequenceClassification', 'GemmaForSequenceClassification', 'Gemma2ForSequenceClassification', 'GPT2ForSequenceClassification', 'GPT2ForSequenceClassification', 'GPTBigCodeForSequenceClassification', 'GPTNeoForSequenceClassification', 'GPTNeoXForSequenceClassification', 'GPTJForSequenceClassification', 'IBertForSequenceClassification', 'JambaForSequenceClassification', 'JetMoeForSequenceClassification', 'LayoutLMForSequenceClassification', 'LayoutLMv2ForSequenceClassification', 'LayoutLMv3ForSequenceClassification', 'LEDForSequenceClassification', 'LiltForSequenceClassification', 'LlamaForSequenceClassification', 'LongformerForSequenceClassification', 'LukeForSequenceClassification', 'MarkupLMForSequenceClassification', 'MBartForSequenceClassification', 'MegaForSequenceClassification', 'MegatronBertForSequenceClassification', 'MistralForSequenceClassification', 'MixtralForSequenceClassification', 'MobileBertForSequenceClassification', 'MPNetForSequenceClassification', 'MptForSequenceClassification', 'MraForSequenceClassification', 'MT5ForSequenceClassification', 'MvpForSequenceClassification', 'NemotronForSequenceClassification', 'NezhaForSequenceClassification', 'NystromformerForSequenceClassification', 'OpenLlamaForSequenceClassification', 'OpenAIGPTForSequenceClassification', 'OPTForSequenceClassification', 'PerceiverForSequenceClassification', 'PersimmonForSequenceClassification', 'PhiForSequenceClassification', 'Phi3ForSequenceClassification', 'PLBartForSequenceClassification', 'QDQBertForSequenceClassification', 'Qwen2ForSequenceClassification', 'Qwen2MoeForSequenceClassification', 'ReformerForSequenceClassification', 'RemBertForSequenceClassification', 'RobertaForSequenceClassification', 'RobertaPreLayerNormForSequenceClassification', 'RoCBertForSequenceClassification', 'RoFormerForSequenceClassification', 'SqueezeBertForSequenceClassification', 'StableLmForSequenceClassification', 'Starcoder2ForSequenceClassification', 'T5ForSequenceClassification', 'TapasForSequenceClassification', 'TransfoXLForSequenceClassification', 'UMT5ForSequenceClassification', 'XLMForSequenceClassification', 'XLMRobertaForSequenceClassification', 'XLMRobertaXLForSequenceClassification', 'XLNetForSequenceClassification', 'XmodForSequenceClassification', 'YosoForSequenceClassification'].\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'label': 'LABEL_0', 'score': 0.9801293015480042}]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "while True:\n",
        "    review = input(\"Enter a movie review (or type 'exit' to stop): \")\n",
        "    if review.lower() == 'exit':\n",
        "        break\n",
        "\n",
        "    # Tokenize the input review\n",
        "    inputs = tokenizer(review, return_tensors=\"pt\", truncation=True, padding=True, max_length=512)\n",
        "    inputs=inputs.to(device)\n",
        "    # Perform sentiment classification\n",
        "    with torch.no_grad():\n",
        "        outputs = model(**inputs)\n",
        "\n",
        "    logits = outputs.logits\n",
        "    probabilities = torch.nn.functional.softmax(logits, dim=-1)\n",
        "    predicted_class = torch.argmax(probabilities, dim=1).item()\n",
        "\n",
        "    sentiment = label_mapping[predicted_class]\n",
        "    print(f\"Sentiment: {sentiment} (Confidence: {probabilities[0][predicted_class].item():.2f})\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gKP1bbExJzZY",
        "outputId": "fa64048a-21d0-4bce-8a9d-288dbf1ec6aa"
      },
      "execution_count": null,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Sentiment: POSITIVE (Confidence: 1.00)\n"
          ]
        }
      ]
    }
  ]
}