BERT_for_Sentiment_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "961b5b49f3cf4828ba3510d721bd57db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_30cb683585664b7a95ddd24de514c103",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a65ea33fe937416bbc29cbebdb025f93",
              "IPY_MODEL_7257f203a0de462789458073993d9948"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "30cb683585664b7a95ddd24de514c103": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "a65ea33fe937416bbc29cbebdb025f93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_527af73b1cdd4f3ba035daceb641ad78",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fdb8ea4ab0814e6da137331241694bdf"
          },
          "model_module_version": "1.5.0"
        },
        "7257f203a0de462789458073993d9948": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8a7c1e4357444b92bb9821a07cd693ae",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 632kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dc7f6589b9be45c5b89a1a088ff5d375"
          },
          "model_module_version": "1.5.0"
        },
        "527af73b1cdd4f3ba035daceb641ad78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "fdb8ea4ab0814e6da137331241694bdf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "8a7c1e4357444b92bb9821a07cd693ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "dc7f6589b9be45c5b89a1a088ff5d375": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "b7ce4c3233df4adfb0e914d5a445a019": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d581b0e01f654ac0b503fd58c5288348",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_18d585f78629424fa2fc61feb2c2fd3a",
              "IPY_MODEL_b0f389b162c94eb1b424e9c50092ec4c"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "d581b0e01f654ac0b503fd58c5288348": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "18d585f78629424fa2fc61feb2c2fd3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_96ec7e0964e24e7da6c64d65ecc195e1",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 361,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 361,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_280acaefa8d84013b50ed9238965b8e0"
          },
          "model_module_version": "1.5.0"
        },
        "b0f389b162c94eb1b424e9c50092ec4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e7f104c91014475cab28e59183a1ec1b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 361/361 [00:03&lt;00:00, 113B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_308be4f1105141a2ac9885758d1dd1f2"
          },
          "model_module_version": "1.5.0"
        },
        "96ec7e0964e24e7da6c64d65ecc195e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "280acaefa8d84013b50ed9238965b8e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "e7f104c91014475cab28e59183a1ec1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "308be4f1105141a2ac9885758d1dd1f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "9b27ad8651b5415f93c401d5cda4a080": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9b298e31e3be4fcb96f83ef18a66b87a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8f66a780bcc84b9fb14794cf3934408a",
              "IPY_MODEL_eb8c20e05f204a97a06f5bd6284a0775"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "9b298e31e3be4fcb96f83ef18a66b87a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "8f66a780bcc84b9fb14794cf3934408a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_47e45c9730574e15a7eeca7fddc99732",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b84700dfac5f4ad4b3100622a6148484"
          },
          "model_module_version": "1.5.0"
        },
        "eb8c20e05f204a97a06f5bd6284a0775": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_af4d474d40af4605be6a399f920b134f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [01:15&lt;00:00, 5.81MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_28bb56ade0004bbd8f41916d4b27df40"
          },
          "model_module_version": "1.5.0"
        },
        "47e45c9730574e15a7eeca7fddc99732": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "b84700dfac5f4ad4b3100622a6148484": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "af4d474d40af4605be6a399f920b134f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "28bb56ade0004bbd8f41916d4b27df40": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/hjiang13/CodeBERT/blob/master/BERT_for_Sentiment_Analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sqg4y371CM7X"
      },
      "source": [
        "# Fine-tuning BERT for Sentiment Analysis\n",
        "[![Portfolio](https://img.shields.io/badge/Portfolio-chriskhanhtran.github.io-blue?logo=GitHub)](https://chriskhanhtran.github.io/)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M2D_810Uruqe"
      },
      "source": [
        "# A - Introduction"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IT93d88Pcepn"
      },
      "source": [
        "In recent years the NLP community has seen many breakthoughs in Natural Language Processing, especially the shift to transfer learning. Models like ELMo, fast.ai's ULMFiT, Transformer and OpenAI's GPT have allowed researchers to achieves state-of-the-art results on multiple benchmarks and provided the community with large pre-trained models with high performance. This shift in NLP is seen as NLP's ImageNet moment, a shift in computer vision a few year ago when lower layers of deep learning networks with million of parameters trained on a specific task can be reused and fine-tuned for other tasks, rather than training new networks from scratch.\n",
        "\n",
        "One of the most biggest milestones in the evolution of NLP recently is the release of Google's BERT, which is described as the beginning of a new era in NLP. In this notebook I'll use the HuggingFace's `transformers` library to fine-tune pretrained BERT model for a classification task. Then I will compare the BERT's performance  with a baseline model, in which I use a TF-IDF vectorizer and a Naive Bayes classifier. The `transformers` library help us quickly and efficiently fine-tune the state-of-the-art BERT model and yield an accuracy rate **10%** higher than the baseline model.\n",
        "\n",
        "**Reference**:\n",
        "\n",
        "To understand **Transformer** (the architecture which BERT is built on) and learn how to implement BERT, I highly recommend reading the following sources:\n",
        "\n",
        "- [The Illustrated BERT, ELMo, and co.](http://jalammar.github.io/illustrated-bert/): A very clear and well-written guide to understand BERT.\n",
        "- [The documentation of the `transformers` library](https://huggingface.co/transformers/v2.2.0/index.html)\n",
        "- [BERT Fine-Tuning Tutorial with PyTorch](http://mccormickml.com/2019/07/22/BERT-fine-tuning/) by [Chris McCormick](http://mccormickml.com/): A very detailed tutorial showing how to use BERT with the HuggingFace PyTorch library.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "slO_rmYgwmmE"
      },
      "source": [
        "# B - Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "31OW0dhozvli"
      },
      "source": [
        "## 1. Load Essential Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_lTXsMK3sNYr"
      },
      "source": [
        "import os\n",
        "import re\n",
        "from tqdm import tqdm\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "%matplotlib inline"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u07WRKnxsX96"
      },
      "source": [
        "## 2. Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j1_Tpie3tGp3"
      },
      "source": [
        "### 2.1. Download Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_tOJXeR9sx57"
      },
      "source": [
        "# Download data\n",
        "import requests\n",
        "request = requests.get(\"https://drive.google.com/uc?export=download&id=1wHt8PsMLsfX5yNSqrt2fSTcb8LEiclcf\")\n",
        "with open(\"data.zip\", \"wb\") as file:\n",
        "    file.write(request.content)\n",
        "\n",
        "# Unzip data\n",
        "import zipfile\n",
        "with zipfile.ZipFile('data.zip') as zip:\n",
        "    zip.extractall('data')"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!ls\n",
        "!ls ./data\n",
        "!head ./data/complaint1700.csv"
      ],
      "metadata": {
        "id": "GfPkMi399AnF",
        "outputId": "bcec4def-22a1-4d3c-f8dd-7f9e3ea65465",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "data  data.zip\tsample_data\n",
            "complaint1700.csv  noncomplaint1700.csv  test_data.csv\n",
            "\"id\",\"airline\",\"tweet\"\n",
            "\"80938\",\"United\",\"@united I'm having issues. Yesterday I rebooked for 24 hours after I was supposed to fly, now I can't log on &amp; check in. Can you help?\"\n",
            "\"10959\",\"United\",\"@united kinda feel like the $6.99 you charge for in flight Wi-Fi is ridiculous. AND it sucks, slow, or doesn't work. #anythingtomakeabuck\"\n",
            "\"130813\",\"SouthWest\",\"Livid in Vegas, delayed, again&amp; again&amp;again, @SouthwestAir decided to cancel a flight and combine two, then waited on crew, now pilots.\"\n",
            "\"146589\",\"United\",\"@united the most annoying man on earth is on my flight. what can you do to help me?\"\n",
            "\"117579\",\"United\",\"@united The last 2 weeks I've flown wit u, you have given me 4 reasons to convince me it was a bad decision. Time 2 go back 2 @SouthwestAir\"\n",
            "\"95632\",\"American\",\"@AmericanAir #AmericanAirlines  Flight 1179 Miami to DC. Gate closes at least 19 minutes before the flight. Agents show up and lie to me.\"\n",
            "\"155716\",\"United\",\"@ABC It appears @united employees have a fear of autism.\"\n",
            "\"146677\",\"JetBlue\",\"Kids have no idea we are heading to WDW tue for an extended vacation. Can't wait to see their face boarding @JetBlue #AKV #DisneySide\"\n",
            "\"33467\",\"United\",\"Four hour delay on my @united flight from dallas to houston. My longest delay i ever had for flight.  Going to be a late night arrival home\"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NVkXcFzrtREn"
      },
      "source": [
        "### 2.2. Load Train Data\n",
        "The train data has 2 files, each containing 1700 complaining/non-complaining tweets. Every tweets in the data contains at least a hashtag of an airline.\n",
        "\n",
        "We will load the train data and label it. Because we use only the text data to classify, we will drop unimportant columns and only keep `id`, `tweet` and `label` columns."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XwjmiM2ktA7d",
        "outputId": "bd5bb97f-833f-4a45-ac88-5d72affa4f57",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "source": [
        " # Load data and set labels\n",
        "data_complaint = pd.read_csv('data/complaint1700.csv')\n",
        "data_complaint['label'] = 0\n",
        "data_non_complaint = pd.read_csv('data/noncomplaint1700.csv')\n",
        "data_non_complaint['label'] = 1\n",
        "\n",
        "# Concatenate complaining and non-complaining data\n",
        "data = pd.concat([data_complaint, data_non_complaint], axis=0).reset_index(drop=True)\n",
        "\n",
        "# Drop 'airline' column\n",
        "data.drop(['airline'], inplace=True, axis=1)\n",
        "\n",
        "# Display 5 random samples\n",
        "data.sample(5)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          id                                              tweet  label\n",
              "219   114827  @DeltaAssist DL3722 - got a call, flt delayed ...      0\n",
              "288    62809  @SouthwestAir you're killing me. 2 bad experie...      0\n",
              "2462   71111                         @JetBlue bad is beautiful.      1\n",
              "1307  137889  @AmericanAir serious issues with miscommunicat...      0\n",
              "1628   26545  @united made me miss a connection, let me canc...      0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8a367e18-3e40-4b90-8cb2-5bfa036890bb\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>tweet</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>219</th>\n",
              "      <td>114827</td>\n",
              "      <td>@DeltaAssist DL3722 - got a call, flt delayed ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>288</th>\n",
              "      <td>62809</td>\n",
              "      <td>@SouthwestAir you're killing me. 2 bad experie...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2462</th>\n",
              "      <td>71111</td>\n",
              "      <td>@JetBlue bad is beautiful.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1307</th>\n",
              "      <td>137889</td>\n",
              "      <td>@AmericanAir serious issues with miscommunicat...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1628</th>\n",
              "      <td>26545</td>\n",
              "      <td>@united made me miss a connection, let me canc...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8a367e18-3e40-4b90-8cb2-5bfa036890bb')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-8a367e18-3e40-4b90-8cb2-5bfa036890bb button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-8a367e18-3e40-4b90-8cb2-5bfa036890bb');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-67afabe6-e114-4658-bfea-17f22bc04bc5\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-67afabe6-e114-4658-bfea-17f22bc04bc5')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-67afabe6-e114-4658-bfea-17f22bc04bc5 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"data\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 44049,\n        \"min\": 26545,\n        \"max\": 137889,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          62809,\n          26545,\n          71111\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"tweet\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"@SouthwestAir you're killing me. 2 bad experiences in a row. 2 hour delay... \\u00f0\\u0178\\u02dc\\u2019\\u00f0\\u0178\\u02dc\\u201d\\u00f0\\u0178\\u02dc\\u00aa #worst\",\n          \"@united made me miss a connection, let me cancel claiming I'd get a credit, now requiring a $200 change fee to use the credit #Unacceptable\",\n          \"@JetBlue bad is beautiful.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          1,\n          0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cp-vfxKZvl6M"
      },
      "source": [
        "We will randomly split the entire training data into two sets: a train set with 90% of the data and a validation set with 10% of the data. We will perform hyperparameter tuning using cross-validation on the train set and use the validation set to compare models."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X4HKAFTbvMwI"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X = data.tweet.values\n",
        "y = data.label.values\n",
        "\n",
        "X_train, X_val, y_train, y_val =\\\n",
        "    train_test_split(X, y, test_size=0.1, random_state=2020)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pErITNxtyNpe"
      },
      "source": [
        "### 2.3. Load Test Data\n",
        "The test data contains 4555 examples with no label. About 300 examples are non-complaining tweets. Our task is to identify their `id` and examine manually whether our results are correct."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_JWXnfBlwyWC",
        "outputId": "99665483-2faa-447d-d520-a538480e9757",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "source": [
        "# Load test data\n",
        "test_data = pd.read_csv('data/test_data.csv')\n",
        "\n",
        "# Keep important columns\n",
        "test_data = test_data[['id', 'tweet']]\n",
        "\n",
        "# Display 5 samples from the test data\n",
        "test_data.sample(5)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          id                                              tweet\n",
              "3474  131450  @united come on! Ridiculous wait for baggage i...\n",
              "20       616  @AmericanAir Should I warn ALL my friends abou...\n",
              "984    39278  @AlaskaAir thanks for ruining my weekend and h...\n",
              "2605   99834  @jetblue We are extremely disappointed and fee...\n",
              "1362   53450  @united flight delayed multiple times. No expl..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-011bbb83-8476-4bcb-99b6-419370603dad\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>tweet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3474</th>\n",
              "      <td>131450</td>\n",
              "      <td>@united come on! Ridiculous wait for baggage i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>616</td>\n",
              "      <td>@AmericanAir Should I warn ALL my friends abou...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>984</th>\n",
              "      <td>39278</td>\n",
              "      <td>@AlaskaAir thanks for ruining my weekend and h...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2605</th>\n",
              "      <td>99834</td>\n",
              "      <td>@jetblue We are extremely disappointed and fee...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1362</th>\n",
              "      <td>53450</td>\n",
              "      <td>@united flight delayed multiple times. No expl...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-011bbb83-8476-4bcb-99b6-419370603dad')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-011bbb83-8476-4bcb-99b6-419370603dad button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-011bbb83-8476-4bcb-99b6-419370603dad');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-b0f5f9f3-c3c1-4923-af6f-11cff1ed2b6f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-b0f5f9f3-c3c1-4923-af6f-11cff1ed2b6f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-b0f5f9f3-c3c1-4923-af6f-11cff1ed2b6f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"test_data\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 51403,\n        \"min\": 616,\n        \"max\": 131450,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          616,\n          53450,\n          39278\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"tweet\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"@AmericanAir Should I warn ALL my friends about my poor experience when I was Navy? Because this is happening: #unsat http://t.co/tsN8wl6ECh\",\n          \"@united flight delayed multiple times. No explanation, no compensation, gate agent just leaves, customer service agent is rude.\",\n          \"@AlaskaAir thanks for ruining my weekend and having me miss my connecting flight in Seattle #customerexperience\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X79dYY3sxDCi"
      },
      "source": [
        "## 3. Set up GPU for training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zi1CoEOL1puh"
      },
      "source": [
        "Google Colab offers free GPUs and TPUs. Since we'll be training a large neural network it's best to utilize these features.\n",
        "\n",
        "A GPU can be added by going to the menu and selecting:\n",
        "\n",
        "`Runtime -> Change runtime type -> Hardware accelerator: GPU`\n",
        "\n",
        "Then we need to run the following cell to specify the GPU as the device."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K7hxtI4l0SUJ",
        "outputId": "f171a628-6148-4252-f678-b935dd1b2e0e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import torch\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    device = torch.device(\"cuda\")\n",
        "    print(f'There are {torch.cuda.device_count()} GPU(s) available.')\n",
        "    print('Device name:', torch.cuda.get_device_name(0))\n",
        "\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are 1 GPU(s) available.\n",
            "Device name: Tesla T4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9j6EbXLs12Kz"
      },
      "source": [
        "# C - Baseline: TF-IDF + Naive Bayes Classifier"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3eWj6qFpA3TE"
      },
      "source": [
        "In this baseline approach, first we will use TF-IDF to vectorize our text data. Then we will use the Naive Bayes model as our classifier.\n",
        "\n",
        "Why Naive Bayse? I have experiemented different machine learning algorithms including Random Forest, Support Vectors Machine, XGBoost and observed that Naive Bayes yields the best performance. In [Scikit-learn's guide](https://scikit-learn.org/stable/tutorial/machine_learning_map/index.html) to choose the right estimator, it is also suggested that Naive Bayes should be used for text data. I also tried using SVD to reduce dimensionality; however, it did not yield a better performance."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZeljUmsqAUpt"
      },
      "source": [
        "## 1. Data Preparation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vU754-QPAwBt"
      },
      "source": [
        "### 1.1. Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4_OzXFcfCBOa"
      },
      "source": [
        "In the bag-of-words model, a text is represented as the bag of its words, disregarding grammar and word order. Therefore, we will want to remove stop words, punctuations and characters that don't contribute much to the sentence's meaning."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "98rwWTSw_dEI",
        "outputId": "94e6f212-ad38-424f-8636-175377b5b199",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import nltk\n",
        "# Uncomment to download \"stopwords\"\n",
        "nltk.download(\"stopwords\")\n",
        "from nltk.corpus import stopwords\n",
        "\n",
        "def text_preprocessing(s):\n",
        "    \"\"\"\n",
        "    - Lowercase the sentence\n",
        "    - Change \"'t\" to \"not\"\n",
        "    - Remove \"@name\"\n",
        "    - Isolate and remove punctuations except \"?\"\n",
        "    - Remove other special characters\n",
        "    - Remove stop words except \"not\" and \"can\"\n",
        "    - Remove trailing whitespace\n",
        "    \"\"\"\n",
        "    s = s.lower()\n",
        "    # Change 't to 'not'\n",
        "    s = re.sub(r\"\\'t\", \" not\", s)\n",
        "    # Remove @name\n",
        "    s = re.sub(r'(@.*?)[\\s]', ' ', s)\n",
        "    # Isolate and remove punctuations except '?'\n",
        "    s = re.sub(r'([\\'\\\"\\.\\(\\)\\!\\?\\\\\\/\\,])', r' \\1 ', s)\n",
        "    s = re.sub(r'[^\\w\\s\\?]', ' ', s)\n",
        "    # Remove some special characters\n",
        "    s = re.sub(r'([\\;\\:\\|•«\\n])', ' ', s)\n",
        "    # Remove stopwords except 'not' and 'can'\n",
        "    s = \" \".join([word for word in s.split()\n",
        "                  if word not in stopwords.words('english')\n",
        "                  or word in ['not', 'can']])\n",
        "    # Remove trailing whitespace\n",
        "    s = re.sub(r'\\s+', ' ', s).strip()\n",
        "\n",
        "    return s"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w8jpfxygCvww"
      },
      "source": [
        "### 1.2. TF-IDF Vectorizer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vbD689AMC-aB"
      },
      "source": [
        "In information retrieval, **TF-IDF**, short for **term frequency–inverse document frequency**, is a numerical statistic that is intended to reflect how important a word is to a document in a collection or corpus. We will use TF-IDF to vectorize our text data before feeding them to machine learning algorithms."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MOQ3X7hPDYhn",
        "outputId": "3571c705-b960-4eea-eb94-bdaf1c863c52",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "%%time\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "# Preprocess text\n",
        "X_train_preprocessed = np.array([text_preprocessing(text) for text in X_train])\n",
        "X_val_preprocessed = np.array([text_preprocessing(text) for text in X_val])\n",
        "\n",
        "# Calculate TF-IDF\n",
        "tf_idf = TfidfVectorizer(ngram_range=(1, 3),\n",
        "                         binary=True,\n",
        "                         smooth_idf=False)\n",
        "X_train_tfidf = tf_idf.fit_transform(X_train_preprocessed)\n",
        "X_val_tfidf = tf_idf.transform(X_val_preprocessed)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 4.74 s, sys: 689 ms, total: 5.43 s\n",
            "Wall time: 5.45 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "arsEHOKzFxdv"
      },
      "source": [
        "## 2. Train Naive Bayes Classifier"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "63HQtpzOInq-"
      },
      "source": [
        "### 2.1. Hyperparameter Tuning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8z5E0Fa6GUyd"
      },
      "source": [
        "We will use cross-validation and AUC score to tune hyperparameters of our model. The function `get_auc_CV` will return the average AUC score from cross-validation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ueXJsrhNGqlS"
      },
      "source": [
        "from sklearn.model_selection import StratifiedKFold, cross_val_score\n",
        "\n",
        "def get_auc_CV(model):\n",
        "    \"\"\"\n",
        "    Return the average AUC score from cross-validation.\n",
        "    \"\"\"\n",
        "    # Set KFold to shuffle data before the split\n",
        "    kf = StratifiedKFold(5, shuffle=True, random_state=1)\n",
        "\n",
        "    # Get AUC scores\n",
        "    auc = cross_val_score(\n",
        "        model, X_train_tfidf, y_train, scoring=\"roc_auc\", cv=kf)\n",
        "\n",
        "    return auc.mean()"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "53MgflYPHNxh"
      },
      "source": [
        "The `MultinominalNB` class only have one hypterparameter - **alpha**. The code below will help us find the alpha value that gives us the highest CV AUC score."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zKatLhhJGzn0",
        "outputId": "af1850cd-c2f3-465b-920c-78351390347c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        }
      },
      "source": [
        "from sklearn.naive_bayes import MultinomialNB\n",
        "\n",
        "res = pd.Series([get_auc_CV(MultinomialNB(i))\n",
        "                 for i in np.arange(1, 10, 0.1)],\n",
        "                index=np.arange(1, 10, 0.1))\n",
        "\n",
        "best_alpha = np.round(res.idxmax(), 2)\n",
        "print('Best alpha: ', best_alpha)\n",
        "\n",
        "plt.plot(res)\n",
        "plt.title('AUC vs. Alpha')\n",
        "plt.xlabel('Alpha')\n",
        "plt.ylabel('AUC')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Best alpha:  1.3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3RVVdrH8e8vgRB6S2gJEIQgIohApAmiYgEs2AVFBVGwIZZxhpl3iuPMOI6OvoodG4oFsY2oqFgQRVEIRZBqKEKoQZQqEMLz/nGP88YYQgK5OSnPZ62s3LvvPifPyVrkxzlnn71lZjjnnHOFFRN2Ac4558oWDw7nnHNF4sHhnHOuSDw4nHPOFYkHh3POuSLx4HDOOVckHhzOlXGShkiaXtx9nTsQDw5X7kn6RNIPkqrk035VnrYTJWXmei9JN0r6RtJOSZmSXpHUvqTqz1XL7ZJMUteS/tnO5ebB4co1SSlAL8CAsw9hFw8Ao4AbgXpAa+A/wBnFU2HhSBJwObAl+O5caDw4XHl3OfAlMA64oigbSkoFrgcGmdnHZrbHzHaZ2Qtmdlc+/S+WlJ6n7WZJk4LX/SUtkrRd0lpJvylCOb2AxkQCbKCkuALqtuAsaYWkzZLukRSTp8+/g7OwlZL65WofKmlxUOMKSSOKUKOrIDw4XHl3OfBC8HW6pIZF2LYPkGlmMwvZ/y3gyCBwfnYJ8GLw+ilghJnVBNoBHxehliuC/U8M3p91kP7nAmlAJ2AAcGWuz7oCS4EE4G7gqeCMBmATcCZQCxgK/K+kTkWo01UAHhyu3JLUE2gOTDSz2cByIn/IC6s+sL6wnc1sF/AmMCj4+alAG2BS0CUbaCuplpn9YGZzCrNfSdWAC4EXzSwbeJWDX676l5ltMbPVwP0/1xT4zsyeMLMc4FkiZzINg2N4x8yWW8Q0YAqRsx3n/suDw5VnVwBTzGxz8P5Ffnm5ah9QOc82lYn8gQf4nsgf1aJ4kf//I30J8J8gUADOB/oD30maJql7Ifd5blDr5OD9C0A/SYkFbLMm1+vvgCa53m/4+UWu2moASOon6UtJWyT9GNSbUMg6XQXhweHKJUlVgYuA3pI2SNoA3Ax0kNQh6LYaSMmzaQsif2gBPgKSJaUV4Ud/ACRKOpZIgPx8mQozm2VmA4AGRG6wT8x/F79yBZE/7KuD43iFSMAVdPbUNNfrZsC6g/2QYNTZa8C/gYZmVodIWKnADV2F48HhyqtzgBygLXBs8HUU8Bn/f5nnZWCopC7BsNvWRMJlAoCZfQs8ArwUDNONkxQvaaCk0fn90OBS0ivAPURGYX0AEGx7qaTaQZ9twP6DHYSkJCL3Ws7MdRwdgH9R8OWq2yTVldSUyKiwlw/2s4A4oAqQBewLbpqfVojtXAXjweHKqyuAZ8xstZlt+PkLeAi4VFIlM3sfGA08A2wl8r/rZ4GxufZzY7DNw8CPRO6TnEvkRvWBvAicArxiZvtytV8GrJK0DbgGuBRAUjNJOyQ1y2dflwHzzGxKnuMYAxwjqd0BangTmA3MA94hcmO+QGa2PTjeicAPRM5oJhW4kauQ5As5OVe+SDIg1cwywq7FlU9+xuGcc65IPDicc84ViV+qcs45VyRRPeOQ1FfSUkkZ+Y1CCW4KTpU0V9J8Sf2D9lMlzZa0IPh+cj7bTpL0TTTrd84592uVorVjSbFERqKcCmQCsyRNMrNFubr9kchTvY9KaktkVEsKsBk4y8zWBaNG3geScu37PGBHYWtJSEiwlJSUwzwi55yrWGbPnr3ZzH71oGnUggPoAmSY2QoASROIzJmTOziMyJw4ALUJHlIys7m5+iwEqkqqYmZ7JNUAbgGGU8gHqFJSUkhPTz94R+ecc/8l6bv82qN5qSqJX057kEmus4bA7cDgYP2DycDIfPZzPjDHzPYE7/8G3Avsyqfvf0kaLildUnpWVtYhlO+ccy4/YY+qGgSMM7NkInPijM89/bOko4k8ITsieH8s0NLM3jjYjs1srJmlmVlaYmJBU/o455wrimgGx1p+OV9OctCW2zCCy01mNgOIJ5hQTVIy8AZwuZktD/p3B9IkrQKmA60lfRKl+p1zzuUjmsExC0iV1CJYdGYgv56+YDWReXiQdBSR4MiSVIfINAmjzezznzub2aNm1sTMUoCewDIzOzGKx+Cccy6PqAVHMEfPDURGRC0mMnpqoaQ7JP28hOetwNWSvgZeAoZY5MGSG4BWwJ8lzQu+GkSrVuecc4VXIR4ATEtLMx9V5ZxzRSNptpn9almBsG+OO+ecK2Oi+RxHuWZmbNi2mxVZO1mRtYOa8ZU5u0MTYmJ8zRvnXPnmwVFIW3bu5Y25a1m6YRtLN+4gY+N2du7N+UWft75ex78v7EDd6nEhVemcc9HnwVEI6au2MPKluazfupuEGnGkNqjJBZ2TadWwJi0TqnNEYg2mLNrA399ezBljPuPBSzrSuXm9sMt2zrmo8OAogJnxxGcr+Nd7S0mqU5VJNxzPMcl18u17efcUOjaty/UvzuGix7/k2KZ1OLZpHTo2q0PPVgnUqeZnIc658sFHVR1Ads5+rn1+Dh8u3kjfoxtx94XHUCu+8kG327Y7m8c+Wc7MlVtYsHYre/btJ7FmFZ4Zchztkmof6iE451yJO9CoKg+OAvzlzW9ISajOkB4pSEW/6Z2ds5853/3AzS/PY9vufTw2uDM9UxOKvB/nnAuDB0eIz3Fs2LqbIc/MJGPTDv59YQfO6Zh3rkfnnCt9/DmOEDWqHc/LI7qTllKXm16ex5iPvqUiBLZzrnzy4CghtatW5tkru3BexyTu+2AZoybMY3d2zsE3dM65UsZHVZWgKpViufeiDrRqWIO731vK6i27ePyyzjSsFR92ac45V2h+xlHCJHHdia14bHBnlm7YTp97pzH20+Xs3bc/7NKcc65QPDhC0rddI965sSddWtTjzslLOP3+T5m6dFPYZTnn3EF5cIToiMQaPD3kOJ4ZehwChj4ziyc/WxF2Wc45VyC/x1EKnHRkA3q0rM/NL8/j7+8s5sdd2dx6WutDenbEOeeizYOjlKhSKZYHB3WidtUFPDQ1gx927eWOAe2I9dl2nXOljAdHKRIbI+48tz11qsXx6CfLWbJhO/84tx1tGtUKuzTnnPuvqN7jkNRX0lJJGZJG5/N5M0lTJc2VNF9S/6D9VEmzJS0Ivp8ctFeT9I6kJZIWSrormvWHQRK/69uG+y7qwIqsHZw5Zjr/fHcxu/buC7s055wDohgckmKBh4F+QFtgkKS2ebr9kcha5B2BgcAjQftm4Cwzaw9cAYzPtc2/zawN0BE4XlK/aB1DmM7rlMzHt57IeZ2SeHzaCvo/8BnLNm4PuyznnIvqGUcXIMPMVpjZXmACMCBPHwN+vg5TG1gHYGZzzWxd0L4QqCqpipntMrOpQZ+9wBwgOYrHEKq61eO4+4IOTBjejZ17czj34c9575sNYZflnKvgohkcScCaXO8zg7bcbgcGS8oEJgMj89nP+cAcM9uTu1FSHeAs4KP8frik4ZLSJaVnZWUd2hGUEt2OqM/bI3uS2rAm1zw/m/umLGX/fp/ryjkXjrCf4xgEjDOzZKA/MF7Sf2uSdDTwL2BE7o0kVQJeAsaYWb4PPpjZWDNLM7O0xMTEqB1ASWlYK54Jw7txYedkxnycwS0T55Gd40+bO+dKXjRHVa0FmuZ6nxy05TYM6AtgZjMkxQMJwCZJycAbwOVmtjzPdmOBb83s/qhUXkrFV47l7guOISWhOve8v5Rtu/fxyKWdiK8cG3ZpzrkKJJpnHLOAVEktJMURufk9KU+f1UAfAElHAfFAVnAZ6h1gtJl9nnsDSX8ncj/kpijWXmpJ4vqTWvH3c9oxdekmLn9qJtt2Z4ddlnOuAolacJjZPuAG4H1gMZHRUwsl3SHp7KDbrcDVkr4mculpiEUWqrgBaAX8WdK84KtBcBbyP0RGac0J2q+K1jGUZoO7NWfMwI7MXfMDI56b7et7OOdKjK8AWMa98NV3/M8b3/Cv89tz8XHNwi7HOVeO+AqA5dSg45rRJaUe/3hnMZu27w67HOdcBeDBUcbFxIg7z2vP7uz9/PWtRWGX45yrADw4yoFWDWow8uRWvDN/PR8u2hh2Oc65cs6Do5wY0bslRzasyZ/e/IYNW/2SlXMuejw4yom4SjHcc+ExbPspm/Me+ZxvfV4r51yUeHCUI8ck1+HlEd3Zm2Nc8NgMZq3aEnZJzrlyyIOjnGmXVJs3rutB/epxXPrkV3zg9zycc8XMg6McalqvGq9e24OjGtXk2udnM2Whz6jrnCs+HhzlVL3qcYy/qitHJ9XmuhfmeHg454qNB0c5Viu+MuOHdaFdEB7ve3g454qBB0c5Vyu+Ms8F4XHt87N5eGqGr+XhnDssHhwVQK34yrxwVVfOPKYJ97y/lGHPzuLHXXvDLss5V0Z5cFQQ1atU4oGBx/K3AUczPWMzZ4yZTsYmf9bDOVd0HhwViCQu657Cq9f0YM++/QwdN4stO/3MwzlXNB4cFVCHpnV44vLObNy2h2uen83efb4ErXOu8Dw4KqiOzepyzwXHMHPlFv70n298ISjnXKFFc81xV8oNODaJjE07ePDjDFIb1uCqXkeEXZJzrgyI6hmHpL6SlkrKkDQ6n8+bSZoqaa6k+ZL6B+2nSpotaUHw/eRc23QO2jMkjZGkaB5DeXfzKa3p164Rd05ezNSlm8IuxzlXBkQtOCTFAg8D/YisET5IUts83f5IZC3yjsBA4JGgfTNwlpm1B64Axufa5lHgaiA1+OobrWOoCGJixL0XdaBNo1rc+OJcn1XXOXdQ0Tzj6AJkmNkKM9sLTAAG5OljQK3gdW1gHYCZzTWzdUH7QqCqpCqSGgO1zOxLi1yUfw44J4rHUCFUi6vEE1ekUaVyLFc9l84PPtLKOVeAaAZHErAm1/vMoC2324HBkjKBycDIfPZzPjDHzPYE22ceZJ8ASBouKV1SelZW1qEdQQWSVKcqj1/WmfU/7ua6F+awZ19O2CU550qpsEdVDQLGmVky0B8YL+m/NUk6GvgXMKKoOzazsWaWZmZpiYmJxVZweda5eV3uOr89M1Z8zyVPfMXmHXvCLsk5VwpFMzjWAk1zvU8O2nIbBkwEMLMZQDyQACApGXgDuNzMlufaZ/JB9ukOw3mdknnoko4sXLeVAQ99zqJ128IuyTlXykQzOGYBqZJaSIojcvN7Up4+q4E+AJKOIhIcWZLqAO8Ao83s8587m9l6YJukbsFoqsuBN6N4DBXSmcc04ZURPcjZb1zw2Bd8vMQXg3LO/b+oBYeZ7QNuAN4HFhMZPbVQ0h2Szg663QpcLelr4CVgSHDT+wagFfBnSfOCrwbBNtcBTwIZwHLg3WgdQ0XWPrk2k244npaJNbjm+TnMWP592CU550oJVYQnhtPS0iw9PT3sMsqkH3ft5cLHZrB+624mDO9Gu6TaYZfknCshkmabWVre9rBvjrtSrk61OMYP60rtqpW54umZrMjaEXZJzrmQeXC4g2pUO57xw7oAMPjJr5iz+oeQK3LOhcmDwxXKEYk1ePbKLkjiwsdmcN8Hy8jO8Vl1nauIPDhcobVLqs27N/ViQIcmjPnoWy549Asyf9gVdlnOuRLmweGKpFZ8Ze67+FgevqQTKzbv5Mpxs9ixZ1/YZTnnSpAHhzskZxzTmMcGd2Z51k5unTiP/fvL/+g851yEB4c7ZMe3SuAP/Y/i/YUbeWhqRtjlOOdKiAeHOyxXHp/CeR2TuO+DZXywyJ8wd64i8OBwh0USd57XnmOSa3P9i3N4JX3NwTdyzpVpHhzusMVXjmXc0C4cl1KX216dzx//s4C9+3yornPllQeHKxb1qsfx7NAujOh9BM9/uZqLx85gw9bdYZflnIsCDw5XbCrFxvD7fkfxyKWdWLphO2c+OJ2vVvjkiM6VNx4crtj1b9+YN68/nlrxlbj0ya945vOVVITJNJ2rKDw4XFSkNqzJf244npPaNOCvby3it6/OJ8ef9XCuXPDgcFFTK74yjw/uzI0nt+KV2Znc9srXHh7OlQOVwi7AlW8xMeKW046kcmwM936wDIB7LuxAbIxCrsw5d6iiesYhqa+kpZIyJI3O5/NmkqZKmitpvqT+QXv9oH2HpIfybDNI0oKg/3uSEqJ5DK54jOyTyq2ntub1uWv5zStfs89n1nWuzIpacEiKBR4G+gFtgUGS2ubp9kciS8p2JLIm+SNB+27gT8Bv8uyzEvAAcJKZHQPMJ7LMrCsDRvZJ5TenteaNuWu57oU57M7OCbsk59whiOYZRxcgw8xWmNleYAIwIE8fA2oFr2sD6wDMbKeZTScSILkp+KouScG266JUv4uCG05O5faz2vLB4o1c/vRMtv6UHXZJzrkiimZwJAG555/IDNpyux0YLCkTmAyMLGiHZpYNXAssIBIYbYGniqleV0KGHN+CBwZ2ZO7qH7j4cX9Q0LmyJuxRVYOAcWaWDPQHxks6YE2SKhMJjo5AEyKXqn5/gL7DJaVLSs/Kyir+yt1hObtDE54Z0oU1W3Zx+v2f8ursTH/Ww7kyIprBsRZomut9ctCW2zBgIoCZzQDigYJudh8b9F1ukb8yE4Ee+XU0s7FmlmZmaYmJiYd2BC6qeqYmMGlkT1Ib1OA3r3zNFc/M8hUFnSsDohkcs4BUSS0kxRG5+T0pT5/VQB8ASUcRCY6CTg/WAm0l/ZwEpwKLi7VqV6JaJtZg4oju/PXso0lftYV+D3zG7O9+CLss51wBohYcZraPyIin94n8cZ9oZgsl3SHp7KDbrcDVkr4GXgKGBGcSSFoF3AcMkZQpqa2ZrQP+CnwqaT6RM5A7o3UMrmTExIgreqTw3qgTqF89jsuf+srnuHKuFFNFuK6clpZm6enpYZfhCmHTtt1c+uRXrPlhF09cnkavVL/M6FxYJM02s7S87WHfHHfuFxrUimfC8G60SKjBsGfT+WTpprBLcs7l4cHhSp36Narw0tVdad2wBtc8P5uZK7eEXZJzLhcPDlcq1akWWRgqqU5Vrhw3i/mZP4ZdknMu4MHhSq36NarwwlXdqFOtMlc8PZNvN24PuyTnHB4crpRrVDue54d1pVJsDJc++RWrv/fnPJwLmweHK/VSEqrz/LCu7M3Zz6VPfcnGbT5FiXNh8uBwZcKRjWry7NAubNmxl8FPfsWWnXvDLsm5CsuDw5UZHZrW4ckrjuO7Lbu4wmfWdS40HhyuTOnesj6PXtqJJRu2ceFjX7Dux5/CLsm5CseDw5U5fY5qyLNDu7D+x92c+8jnLF6/LeySnKtQPDhcmdSjVQITr+mOEBc9NoMvlm8OuyTnKgwPDldmHdW4Fq9f14PGdeIZ8swspi7x6UmcKwkeHK5Ma1KnKi8P707rhjUYPj6d977ZEHZJzpV7HhyuzKtbPY4XrupGu6TaXP/iHN762pehdy6aPDhcuVC7amXGD+tK5+Z1GTVhLhNmrg67JOfKLQ8OV27UqFKJZ4d2oVdqIqNfX8DYT5eHXZJz5dIBg0PS6ZIuyKf9AkmnRrcs5w5N1bhYnrg8jTOOacydk5dw93tLqAiLlTlXkgo64/gzMC2f9k+AOwqzc0l9JS2VlCFpdD6fN5M0VdJcSfMl9Q/a6wftOyQ9lGebOEljJS2TtETS+YWpxVUccZViGDOwI4O6NOWRT5Zz/Ytz2LxjT9hlOVduFBQcVcwsK2+jmW0Gqh9sx5JigYeBfkBbYJCktnm6/ZHIWuQdgYHAI0H7buBPwG/y2fX/AJvMrHWw3/zCzVVwsTHiznPb89u+R/Lhok2cet80/jN3rZ99OFcMCgqOWpIq5W2UVBmoWoh9dwEyzGyFme0FJgAD8vQxoFbwujawDsDMdprZdCIBkteVwD+DfvuDIHPuVyRx3YmteOfGnqQkVOeml+dx9XPpPkGic4epoOB4HXhC0n/PLiTVAB4LPjuYJGBNrveZQVtutwODJWUCk4GRBe1QUp3g5d8kzZH0iqSGB+g7XFK6pPSsrF+dOLkKJLVhTV69pgd/OrMtny7bTP8HPmPWKl+O1rlDVVBw/BHYCHwnabakOcBKICv4rDgMAsaZWTLQHxgvqaCaKgHJwBdm1gmYAfw7v45mNtbM0swsLTExsZjKdWVVbIwY1rMFr1/Xg/jKMQwc+yWPfJLB/v1+6cq5ojrgH2kz22dmo4GmwBDgCqCZmY02s8LMZ7022PZnyUFbbsOAicHPmwHEAwkF7PN7YBf/f8bzCtCpELU4B0C7pNq8NbIn/do14u73ljLoiS99VUHniqig4bjnSTqPyM3tVKAVkCapZiH3PQtIldRCUhyRm9+T8vRZDfQJft5RRILjgNeVLHJn8y3gxKCpD7CokPU4B0DN+Mo8OKgjd19wDIvWbeP0+z/l2S9W+dmHc4WkA40ykfRMPs31gGOAYWb28UF3Hhleez8QCzxtZv+QdAeQbmaTglFWTwA1iNwo/62ZTQm2XUXkxnkc8CNwmpktktQcGA/UIRIyQ82swMeE09LSLD09/WDlugpo/dafGP3aAqYty6J360QeHdyJanG/GhPiXIUkabaZpf2qvajDE4M/3BPNrGtxFRdtHhyuIGbG81+t5i9vfsOxTevwzJAu1K5WOeyynAvdgYKjyFOOmNl3gP+rcuWGJC7r1pxHLu3EN2u3cfHYGWzant9IcOccHEJwSGoD+GO4rtzp264xTw85jtVbdnHhYzNYuXln2CU5VyoVdHP8LUmT8nxNB94Bbim5Ep0rOT1TE3j+qq5s+ymbcx7+3FcWdC4fBd0c752nyYAtRG6QX2xm10e5tmLj9zhcUa3+fhfDnp3Fys07uWNAOy7p2izskpwrcUW+x2Fm037+ArYBZwFvA38FFketUudKgWb1q/HadT04vlUCf3hjAbdPWsi+nP1hl+VcqXDAcYeSWhN5snsQsBl4mcgZykklVJtzoaoVX5mnrkjjzslLePrzlWRs2sFDl3SkTrW4sEtzLlQF3RxfApwMnGlmPc3sQSCnZMpyrnSoFBvDn89qy93nH8NXK7/nnIc/J2PT9rDLci5UBQXHecB6YKqkJyT1AVQyZTlXulx0XFNeurobO/bs45yHv/B1zV2FVtA9jv+Y2UCgDTAVuAloIOlRSaeVVIHOlRZpKfV484aetG5Yg5EvzeX3ry9gd7afhLuK56DPcQRrY7xoZmcRmahwLvC7qFfmXCmUVKcqL4/ozjW9W/LSzNWc8/DnLNmwLeyynCtRRXoA0Mx+CKYr7xOtgpwr7SrHxjC6XxvGDT2OrO17OHPMdO6bspQ9+/zsw1UMRX5y3DkXceKRDfjglt6c3aEJYz7OoP8Dn5HuC0S5CsCDw7nDUK96HPddfCzjhh7H7uz9XPDYDH736nx+8OVpXTnmweFcMTjxyAZMufkERpxwBK/NyeTkez9h4qw1vsaHK5c8OJwrJtWrVOL3/Y/inRt70apBDX772nwuf3om6378KezSnCtWHhzOFbMjG9Vk4oju/OPcdsxZ/QOn3/8pr83OpKhr3zhXWnlwOBcFkri0a3PeHdWLNo1qcusrX3PVs+ms2eLrm7uyL6rBIamvpKWSMiSNzufzZpKmSporaX6w1CyS6gftOyQ9dIB9T5L0TTTrd+5wNa9fnQnDu/M//Y/ii+Xfc8p903jwo2996K4r06IWHJJigYeBfkBbYFCwxnhufySyDG1HYCDwSNC+G/gT8JsD7Ps8YEc06nauuMXGiKtPOIKPbu1Nn6MacO8Hy+h7/2d8s3Zr2KU5d0iiecbRBcgwsxVmtheYAAzI08eAWsHr2sA6+O/T6tOJBMgvSKpBZCGpv0ercOeioUmdqjxyaWeeu7ILu7NzOP/RL3glfU3YZTlXZNEMjiQg97+KzKAtt9uBwZIygcnAyELs92/AvUCBF4slDZeULik9Kyur0EU7F20ntE7krZE96dy8Lre9Op8/vLHAL125MiXsm+ODgHFmlgz0B8ZLKmg522OBlmb2xsF2HEyNkmZmaYmJicVXsXPFIKFGFZ67sgvXntiSF79azSn3TeP1OZnk+HMfrgyIZnCsBZrmep8ctOU2DJgIYGYzgHggoYB9dgfSJK0CpgOtJX1STPU6V6Iqxcbwu75tGD+sCzWrVOaWiV/T9/5P+XDRxrBLc65A0QyOWUCqpBaS4ojc/J6Up89qoA+ApKOIBMcBryuZ2aNm1sTMUoCewDIzOzEKtTtXYnqlJvL2yJ48fEkncsy46rl0fvvq1+zauy/s0pzL1wGXjj1cZrZP0g3A+0As8LSZLZR0B5BuZpOAW4EnJN1M5Eb5EAuekgrOKmoBcZLOAU4zs0XRqte5MMXEiDOOacxpRzfkgQ+/5eFPMkhf9QNjBnWkXVLtsMtz7hdUEZ5mTUtLs/T09LDLcK7Qvli+mVte/prvd+5h5MmpjOh9BFUqxYZdlqtgJM02s7S87WHfHHfO5aNHywTeHdWL049uxH0fLKPfA58xY/n3YZflHODB4VypVbd6HA9d0olxQ48jO2c/g574klsmzmOLT9nuQubB4Vwpd+KRDZhyU2+uP6klk+ato8+9n/BK+hqfNNGFxoPDuTKgalwst53ehsmjenFEYg1ue3U+lzzxlU+a6ELhweFcGdK6YU1eGdGdO89tzzdrt9L/gc94c17ex6Ociy4PDufKmJgYcUnXZkwe1YvWjWoyasI8bnl5Htt3Z4ddmqsgPDicK6Oa1qvGy8O7MapPKv+Zt5Zed0/lfz9Y5jfPXdT5cxzOlQPzM3/kwY8z+GDRRqpWjmVQl2bcdGoqteIrh12aK8MO9ByHB4dz5ci3G7fz2LQVvDE3k8SaVfjHOe05pW3DsMtyZZQ/AOhcBZDasCb3XtSBN647nrrV4rjquXRufGkum3fsCbs0V454cDhXDnVoWodJN/Tk5lNa8+436znpnk8Y++lyX/fDFQsPDufKqbhKMYw6JZV3R51AWkpd7py8hNP+16dtd4fPg8O5cq5Vgxo8M7QL44YeR+XYGK56Lp3rXpjNpu2/WpnZuULx4HCugjjxyAa8O6oXt51+JB8u3sQp905j4iyfusQVnQeHcxVI5TyUwpQAABHuSURBVNgYrj+pFe+O6kWbRrX47WvzuejxGSxaty3s0lwZ4sHhXAXUMrEGE4Z3467z2pOxaQdnPvgZf3nzG7bu8qfP3cF5cDhXQcXEiIFdmjH1NydyadfmjP/yO3re/TH3TVnKD/70uStAVINDUl9JSyVlSBqdz+fNJE2VNFfSfEn9g/b6QfsOSQ/l6l9N0juSlkhaKOmuaNbvXEVQp1ocfzunHW+P7MXxLRMY83EGx//rY+6cvJgfd3mAuF+L2pPjkmKBZcCpQCYwCxiUe91wSWOBuWb2qKS2wGQzS5FUHegItAPamdkNQf9qQFczmyopDvgIuNPM3i2oFn9y3LnCW7ZxO49MzWDS1+uoXbUyt53ehouPa0psjMIuzZWwMJ4c7wJkmNkKM9sLTAAG5OljQK3gdW1gHYCZ7TSz6cAvxgua2S4zmxq83gvMAZKjdwjOVTytG9bk/oEdeXtkL1Ib1uQPbyxgwMPTmbP6h7BLc6VENIMjCViT631m0Jbb7cBgSZnAZGBkYXcuqQ5wFpGzjvw+Hy4pXVJ6VlZWUep2zgFtm9Ti5eHdGDOoI5u37+X8R7/gL29+w449+8IuzYUs7Jvjg4BxZpYM9AfGSzpoTZIqAS8BY8xsRX59zGysmaWZWVpiYmKxFu1cRSGJszs04cNbe3NF9xSe+/I7Tr1vGlMWbvDnPyqwaAbHWqBprvfJQVtuw4CJAGY2A4gHEgqx77HAt2Z2fzHU6Zw7iBpVKnH72Ufz2rU9qBVfmeHjZ3PR4zOYuXJL2KW5EEQzOGYBqZJaBDeyBwKT8vRZDfQBkHQUkeAo8LqSpL8TuR9yU7FX7JwrUKdmdXn7xp787Zx2fPf9Li56fAaXPz2Theu2hl2aK0FRXY8jGF57PxALPG1m/5B0B5BuZpOCkVRPADWI3Cj/rZlNCbZdReTGeRzwI3AasI3IfZMlwM/zRD9kZk8WVIePqnKu+P20N4fnZqzi0WnL2fpTNhd2TuY3px1Jg1rxYZfmiokv5OTB4VxUbN2VzYMff8uzM1ZROTaGa3q35KpeLagWVyns0txh8oWcnHNRUbtaZf54Zls+uLk3J6Qmct8Hyzjxnk+YMHM1+3L2h12eiwIPDudcsUhJqM5jl3Xm1Wu6k1y3KqNfX0C/Bz7jnfnr2b+//F/ZqEg8OJxzxSotpR6vXduDxwZ3IseM61+cw+n3f8qb89aS4wFSLnhwOOeKnST6tmvMBzf3ZsygjgCMmjCPvvd/ykeLN/ozIGWcB4dzLmpiYyIPEL5/0wk8fEkn9u03hj2bzsCxXzLXpzAps3xUlXOuxGTn7GfCzNXc/+G3fL9zL60b1uCM9k04s0NjWibWCLs8l4cPx/XgcK7U2LFnH2/MyeSt+euZtWoLZtC1RT1G9Umle8v6SD4Tb2ngweHB4VyptHHbbibNW8eT01ewcdseOjevy82ntKZnamFmH3LR5MHhweFcqbY7O4dX0tfw6CfLWbd1NwOObcKfz2xL/RpVwi6twvIHAJ1zpVp85Vgu657C1NtOZFSfVCYvWM8p903jP3PX+iisUsaDwzlXqlSpFMvNp7bmnRt70bx+dW56eR4Dx37JN2t9IsXSwoPDOVcqtW5Yk9eu7cHfz2lHxqYdnPXQdG6ZOI/1W38Ku7QKz4PDOVdqxcaIwd2aM/W2ExlxQkve/no9J/97Gg98+C0/7c0Ju7wKy4PDOVfq1YqvzOh+bfjo1t6c3KYB//vhMvrc+wlvzlvr82CFwIPDOVdmNK1XjYcv7cTLw7tRt3ocoybMo98DnzF5gU+kWJI8OJxzZU7XI+oz6Yae3H/xsWTv3891L8yh3wOfMenrdT6Vewnw5zicc2Vazn7j7fnreOCjb1mRtZPkulW5utcRXJiW7ItJHaZQnuOQ1FfSUkkZkkbn83kzSVMlzZU0P1hqFkn1g/Ydkh7Ks01nSQuCfY6Rz03gXIUWGyMGHJvEhzf35vHLOtOgZhX+Mmkh3f/5MX9+8xsWZG7150CKWdTOOCTFAsuAU4FMYBYwyMwW5eozFphrZo8G649PNrMUSdWBjkA7oJ2Z3ZBrm5nAjcBXwGRgjJm9W1AtfsbhXMWSvmoLz874jvcXbmDvvv20aVSTczomcUb7xjStVy3s8sqMA51xRPM8rguQYWYrggImAAOARbn6GFAreF0bWAdgZjuB6ZJa5d6hpMZALTP7Mnj/HHAOUGBwOOcqlrSUeqSl1GPrT9m89fU6XpuTyV3vLuGud5fQIbk2/ds3ps9RDWmZWN0nVDwE0QyOJGBNrveZQNc8fW4HpkgaCVQHTinEPjPz7DMpv46ShgPDAZo1a1boop1z5UftqpUZ3K05g7s1Z82WXUxesJ6356/nn+8u4Z/vLqF5/Wr0adOQ8zol0S6pdtjllhlh3zkaBIwzs3sldQfGS2pnZoc9LMLMxgJjIXKp6nD355wr25rWq8aI3i0Z0bsl6378iY+WbOLjxRt5/qvvePrzlXRIrs2gLs04+9gmflP9IKJ5c3wt0DTX++SgLbdhwEQAM5sBxAMFzaW8NthPQft0zrkCNalTlcu6NeeZoV2Y9YdT+MtZbdm1N4fRry/gxHsiDxb6DfUDi2ZwzAJSJbWQFAcMBCbl6bMa6AMg6SgiwZF1oB2a2Xpgm6RuwWiqy4E3o1G8c65iqF2tMkOPb8GUm09gwvBuNKodz6gJ87h47Jcs2bAt7PJKpag+xxEMr70fiAWeNrN/SLoDSDezScFIqieAGkRulP/WzKYE264icuM8DvgROM3MFklKA8YBVYncFB9pBzkIH1XlnCusnP3GxPQ13P3eErb+lE2/9o25tnfLCnkPxBdy8uBwzhXBDzv38vinK3jhy+/YvmcfvVITuKZ3S3pUoKVtPTg8OJxzh2Db7mxe/Go1T01fSdb2PRyTXJtre7fktKMbERtTvgPEg8ODwzl3GHZn5/DG3LU8Pm05q77fRYuE6gw/4QjO65RElUqxYZcXFR4cHhzOuWKQs994f+EGHv1kOQvWbqVBzSpc2bMF53ZMomGt+LDLK1YeHB4czrliZGZ8sfx7Hv1kOdMzNgPQplFNeh+ZSP92jenQtE7IFR4+Dw4PDudclHy7cTsfL9nEtGVZzFq1hewc46QjE7n1tCPL9GgsDw4PDudcCdi+O5vnv1zNY9OWs/WnbE4/uiE3nJRK++SyFyAeHB4czrkStG13Nk9PX8lTn61k+5599GhZnxG9W3JCakKZGc7rweHB4ZwLwbbd2UyYGRnOu3HbHo5uUotbT2vNSUc2KPUB4sHhweGcC9Hefft5c95aHvw4g9VbdtGxWR1uPqU1PVslEFNKnwfx4PDgcM6VAtk5+3l1diZjPvqW9Vt307h2PGce05izOyTRLqlWqToL8eDw4HDOlSK7s3N4f+EG3vp6HdOWZZGdYxzVuBZDj0/h7A5NiK8c/kOFHhweHM65UurHXXt5Z8F6nvviO5Zu3E796nFc2rUZl/dIIaFGldDq8uDw4HDOlXJmxozl3/P056v4aMlG4mJjOL9zMlf3OoIWCdVLvJ4w1hx3zjlXBJLo0SqBHq0SWJ61gyc/W8GrszN5aeZqjm+ZwPmdkzj96Eahr1DoZxzOOVeKbdq+mxe+XM3rczNZs+UnqsfF0q99Y87rmES3I+pHdUSWX6ry4HDOlWH79xuzVm3htTmZTF6wgR179tG4djwDjk1iUJemNK9f/JeyQgkOSX2BB4isAPikmd2V5/NmwLNAnaDPaDObHHz2eyJrkucAN5rZ+0H7zcBVRFYMXAAMNbPdBdXhweGcK092Z+fwwaKNvDF3LdOWZZGz3+iVmsBl3ZpzcpsGVIotnlXBSzw4JMUCy4BTgUwia5APMrNFufqMBeaa2aPBMrKTzSwleP0S0AVoAnwItAYaAdOBtmb2k6SJwTbjCqrFg8M5V15t3LabCTPX8NLM1WzYtpvEmlU4t2MS53VKok2jWoe17wMFR/HEUv66ABlmtsLM9gITgAF5+hiRdcUBagPrgtcDgAlmtsfMVgIZwf4gckO/qqRKQLVc2zjnXIXTsFY8o05JZfrvTmLsZZ3p2LQOT09fSd/7P+OMMZ+RtX1Psf/MaN6aTwLW5HqfCXTN0+d2YIqkkUB14JRc236ZZ9skM5sh6d/AauAnYIqZTYlC7c45V6ZUio3htKMbcdrRjdiycy+T5q1lxorvSagRV+w/K5pnHIUxCBhnZslAf2C8pAPWJKkukbORFkQuYVWXNPgAfYdLSpeUnpWVFYXSnXOudKpXPY4hx7fg8cvSojKFSTSDYy3QNNf75KAtt2HARAAzmwHEAwkFbHsKsNLMsswsG3gd6JHfDzezsWaWZmZpiYmJxXA4zjnnILrBMQtIldRCUhwwEJiUp89qoA+ApKOIBEdW0G+gpCqSWgCpwMygfzdJ1RSJ0T7A4igeg3POuTyido/DzPZJugF4n8hQ26fNbKGkO4B0M5sE3Ao8EQyxNWCIRYZ5LQxGTC0C9gHXm1kO8JWkV4E5QftcYGy0jsE559yv+QOAzjnn8hXGcFznnHPlkAeHc865IvHgcM45VyQeHM4554qkQtwcl5QFfBd2HYcpAdgcdhGljP9Ofsl/H7/kv49fK+rvpLmZ/epBuAoRHOWBpPT8RjdUZP47+SX/ffyS/z5+rbh+J36pyjnnXJF4cDjnnCsSD46yw5+Q/zX/nfyS/z5+yX8fv1YsvxO/x+Gcc65I/IzDOedckXhwOOecKxIPjlJOUlNJUyUtkrRQ0qiwayoNJMVKmivp7bBrCZukOpJelbRE0mJJ3cOuKWySbg7+vXwj6SVJ8WHXVNIkPS1pk6RvcrXVk/SBpG+D73UPZd8eHKXfPuBWM2sLdAOul9Q25JpKg1H4Wiw/ewB4z8zaAB2o4L8XSUnAjUCambUjsqzDwHCrCsU4oG+ettHAR2aWCnwUvC8yD45SzszWm9mc4PV2In8UksKtKlySkoEzgCfDriVskmoDJwBPAZjZXjP7MdyqSoVKQFVJlYBqwLqQ6ylxZvYpsCVP8wDg2eD1s8A5h7JvD44yRFIK0BH4KtxKQnc/8Ftgf9iFlAItiKya+Uxw6e5JSdXDLipMZrYW+DeRFUPXA1vNbEq4VZUaDc1sffB6A9DwUHbiwVFGSKoBvAbcZGbbwq4nLJLOBDaZ2eywayklKgGdgEfNrCOwk0O8/FBeBNftBxAJ1SZAdUmDw62q9AlWWz2k5zE8OMoASZWJhMYLZvZ62PWE7HjgbEmrgAnAyZKeD7ekUGUCmWb281noq0SCpCI7BVhpZllmlg28DvQIuabSYqOkxgDB902HshMPjlJOkohcv15sZveFXU/YzOz3ZpZsZilEbnh+bGYV9n+TZrYBWCPpyKCpD7AoxJJKg9VAN0nVgn8/fajgAwZymQRcEby+AnjzUHbiwVH6HQ9cRuR/1vOCr/5hF+VKlZHAC5LmA8cCd4ZcT6iCs69XgTnAAiJ/5yrc9COSXgJmAEdKypQ0DLgLOFXSt0TOzO46pH37lCPOOeeKws84nHPOFYkHh3POuSLx4HDOOVckHhzOOeeKxIPDOedckXhwOFeMJJ0jySS1Cd6n5J6d9ADbHLSPc6WJB4dzxWsQMD347ly55MHhXDEJ5hPrCQwjn2m8JQ2R9KakT4L1EP6S6+NYSU8Ea0hMkVQ12OZqSbMkfS3pNUnVSuZonDswDw7nis8AIutiLAO+l9Q5nz5dgPOBY4ALJaUF7anAw2Z2NPBj0AfgdTM7zsx+XmdjWFSPwLlC8OBwrvgMIjLxIsH3/C5XfWBm35vZT0Qm3+sZtK80s3nB69lASvC6naTPJC0ALgWOjkrlzhVBpbALcK48kFQPOBloL8mIrDpnwMN5uuad4+fn93tyteUAVYPX44BzzOxrSUOAE4uvaucOjZ9xOFc8LgDGm1lzM0sxs6bASqBpnn6nBus+VyWy+trnB9lvTWB9MLX+pcVetXOHwIPDueIxCHgjT9trwO/ztM0M2ucDr5lZ+kH2+yciKz5+DiwphjqdO2w+O65zJSS41JRmZjeEXYtzh8PPOJxzzhWJn3E455wrEj/jcM45VyQeHM4554rEg8M551yReHA455wrEg8O55xzRfJ/MSqXvDZYRG4AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JaATcy1nIeE9"
      },
      "source": [
        "### 2.2. Evaluation on Validation Set"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ne-eoqM4Muna"
      },
      "source": [
        "To evaluate the performance of our model, we will calculate the accuracy rate and the AUC score of our model on the validation set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qS2gb-9mJK2w"
      },
      "source": [
        "from sklearn.metrics import accuracy_score, roc_curve, auc\n",
        "\n",
        "def evaluate_roc(probs, y_true):\n",
        "    \"\"\"\n",
        "    - Print AUC and accuracy on the test set\n",
        "    - Plot ROC\n",
        "    @params    probs (np.array): an array of predicted probabilities with shape (len(y_true), 2)\n",
        "    @params    y_true (np.array): an array of the true values with shape (len(y_true),)\n",
        "    \"\"\"\n",
        "    preds = probs[:, 1]\n",
        "    fpr, tpr, threshold = roc_curve(y_true, preds)\n",
        "    roc_auc = auc(fpr, tpr)\n",
        "    print(f'AUC: {roc_auc:.4f}')\n",
        "\n",
        "    # Get accuracy over the test set\n",
        "    y_pred = np.where(preds >= 0.5, 1, 0)\n",
        "    accuracy = accuracy_score(y_true, y_pred)\n",
        "    print(f'Accuracy: {accuracy*100:.2f}%')\n",
        "\n",
        "    # Plot ROC AUC\n",
        "    plt.title('Receiver Operating Characteristic')\n",
        "    plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)\n",
        "    plt.legend(loc = 'lower right')\n",
        "    plt.plot([0, 1], [0, 1],'r--')\n",
        "    plt.xlim([0, 1])\n",
        "    plt.ylim([0, 1])\n",
        "    plt.ylabel('True Positive Rate')\n",
        "    plt.xlabel('False Positive Rate')\n",
        "    plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xnCfezJSM-41"
      },
      "source": [
        "By combining TF-IDF and the Naive Bayes algorithm, we achieve the accuracy rate of **72.65%** on the validation set. This value is the baseline performance and will be used to evaluate the performance of our fine-tune BERT model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YwVGbLHLIwpl",
        "outputId": "4e235965-2d3f-4396-a329-d97dd9148c5e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 329
        }
      },
      "source": [
        "# Compute predicted probabilities\n",
        "nb_model = MultinomialNB(alpha=1.8)\n",
        "nb_model.fit(X_train_tfidf, y_train)\n",
        "probs = nb_model.predict_proba(X_val_tfidf)\n",
        "\n",
        "# Evaluate the classifier\n",
        "evaluate_roc(probs, y_val)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "AUC: 0.8451\n",
            "Accuracy: 75.59%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd5hU9fXH8fcBaSKgojFKUaJYwIKwETvYAUFUFMEGCmKvaDTRREOIxhJbgkqR4M+CXURFMQqIqKggoBRRBKUoEREVFJByfn9877rDujs77O7MnZn9vJ5nnp2Ze+feM3d358y33HPN3RERESlNtbgDEBGR7KZEISIiSSlRiIhIUkoUIiKSlBKFiIgkpUQhIiJJKVHIZjGzWWbWPu44soWZ/cnMhsW07xFmNjCOfVc2MzvDzF4t52v1N5lmShQ5zMw+N7PVZrbKzJZGHxxbpXOf7t7S3Sekcx+FzKyWmd1iZguj9/mpmV1jZpaJ/ZcQT3szW5z4nLvf7O5907Q/M7PLzGymmf1oZovN7Ckz2ycd+ysvM7vJzB6pyDbc/VF3PzaFff0qOWbyb7KqUqLIfV3cfSugFbA/8MeY49lsZrZFKYueAo4COgH1gLOAfsA9aYjBzCzb/h/uAS4HLgO2BXYHRgHHV/aOkvwO0i7OfUuK3F23HL0BnwNHJzy+DXgp4fGBwNvAd8AMoH3Csm2B/wBfAiuAUQnLOgPTo9e9DexbfJ/ATsBqYNuEZfsD3wA1osfnAnOi7Y8Fdk5Y14GLgU+BBSW8t6OANUCTYs+3BTYAu0WPJwC3AO8BPwDPF4sp2TGYAPwdeCt6L7sB50QxrwTmA+dH69aN1tkIrIpuOwE3AY9E6+wSva9ewMLoWFyfsL86wEPR8ZgD/AFYXMrvtnn0Pg9I8vsfAQwCXorifRfYNWH5PcCi6LhMBQ5LWHYT8DTwSLS8L3AA8E50rL4C/g3UTHhNS+C/wLfA/4A/AR2An4F10TGZEa3bAHgw2s4SYCBQPVrWOzrmdwHLo2W9gUnRcouWfR3F9hGwN+FLwrpof6uAF4r/HwDVo7g+i47JVIr9DelWjs+auAPQrQK/vE3/QRpH/1D3RI8bRf+EnQgtx2Oix9tHy18CngC2AWoA7aLn94/+QdtG/3S9ov3UKmGf44DzEuK5HXggut8VmAfsBWwB3AC8nbCuRx862wJ1Snhv/wDeKOV9f0HRB/iE6INob8KH+TMUfXCXdQwmED7QW0Yx1iB8W981+rBqB/wEtI7Wb0+xD3ZKThRDCUlhP2AtsFfie4qOeWPgw+LbS9juBcAXZfz+R0Tv54Ao/keBxxOWnwk0jJb1B5YCtRPiXgecGB2bOkAbQmLdInovc4ArovXrET70+wO1o8dtix+DhH0/BwyOfie/ISTywt9Zb2A9cGm0rzpsmiiOI3zAbx39HvYCdkx4zwOT/B9cQ/g/2CN67X5Aw7j/V3P9FnsAulXglxf+QVYRvjk58DqwdbTsWuDhYuuPJXzw70j4ZrxNCdu8H/hbsefmUpRIEv8p+wLjovtG+PZ6ePT4ZaBPwjaqET50d44eO3Bkkvc2LPFDr9iyyUTf1Akf9v9IWNaC8I2zerJjkPDaAWUc41HA5dH99qSWKBonLH8P6BHdnw8cl7Csb/HtJSy7HphcRmwjgGEJjzsBHydZfwWwX0LcE8vY/hXAc9H9nsC0Utb75RhEj3cgJMg6Cc/1BMZH93sDC4ttozdFieJI4BNC0qpWwntOlijmAl3T8f9WlW/Z1icrm+9Ed69H+BDbE9guen5n4FQz+67wBhxKSBJNgG/dfUUJ29sZ6F/sdU0I3SzFPQMcZGY7AocTks+bCdu5J2Eb3xKSSaOE1y9K8r6+iWItyY7R8pK28wWhZbAdyY9BiTGYWUczm2xm30brd6LomKZqacL9n4DCCQY7Fdtfsve/nNLffyr7wsyuNrM5ZvZ99F4asOl7Kf7edzezF6OJET8ANyes34TQnZOKnQm/g68SjvtgQsuixH0ncvdxhG6vQcDXZjbEzOqnuO/NiVNSpESRJ9z9DcK3rTuipxYRvk1vnXCr6+7/iJZta2Zbl7CpRcDfi71uS3cfWcI+VwCvAqcBpxNaAJ6wnfOLbaeOu7+duIkkb+k1oK2ZNUl80szaEj4MxiU8nbhOU0KXyjdlHINfxWBmtQjJ7w5gB3ffGhhDSHBlxZuKrwhdTiXFXdzrQGMzKyjPjszsMMIYSHdCy3Fr4HuK3gv8+v3cD3wMNHf3+oS+/sL1FwG/K2V3xbeziNCi2C7huNd395ZJXrPpBt3vdfc2hBbi7oQupTJfF+171zLWkc2kRJFf7gaOMbP9CIOUXczsODOrbma1o+mdjd39K0LX0H1mto2Z1TCzw6NtDAUuMLO20UygumZ2vJnVK2WfjwFnA6dE9ws9APzRzFoCmFkDMzs11Tfi7q8RPiyfMbOW0Xs4MHpf97v7pwmrn2lmLcxsS2AA8LS7b0h2DErZbU2gFrAMWG9mHYHEKZv/AxqaWYNU30cxTxKOyTZm1gi4pLQVo/d3HzAyirlmFH8PM7suhX3VI4wDLAO2MLO/AGV9K69HGDxeZWZ7AhcmLHsR2NHMroimLdeLkjaE47JL4ayx6O/rVeCfZlbfzKqZ2a5m1i6FuDGz30d/fzWAHwmTGjYm7Ku0hAWhy/JvZtY8+vvd18waprJfKZ0SRR5x92XA/wF/cfdFhAHlPxE+LBYRvpUV/s7PInzz/pgweH1FtI0pwHmEpv8KwoB07yS7HU2YobPU3WckxPIccCvweNSNMRPouJlvqRswHniFMBbzCGEmzaXF1nuY0JpaShhovSyKoaxjsAl3Xxm99knCez89en+Fyz8GRgLzoy6VkrrjkhkALAYWEFpMTxO+eZfmMoq6YL4jdKmcBLyQwr7GEo7bJ4TuuDUk7+oCuJrwnlcSvjA8UbggOjbHAF0Ix/lT4Iho8VPRz+Vm9kF0/2xC4p1NOJZPk1pXGoSENjR63ReEbrjbo2UPAi2i4z+qhNfeSfj9vUpIeg8SBsulAqyop0Ak95jZBMJAaixnR1eEmV1IGOhO6Zu2SFzUohDJEDPb0cwOibpi9iBMNX0u7rhEypK2RGFmw83sazObWcpyM7N7zWyemX1oZq3TFYtIlqhJmP2zkjAY/zxhHEIkq6Wt6ykaHF0F/J+7713C8k6EvuZOhJO77nH3tsXXExGReKWtReHuEwlz50vTlZBE3N0nA1tH8/FFRCSLxFmMqxGbzsJYHD33VfEVzawfoc4LdevWbbPnnntmJEARkcoydy6sXg11MjwHa4e1X7DV+u+Y4eu/cffty7ONnKja6O5DgCEABQUFPmXKlJgjEpFcMmQIPPZY2eulU/XqcOihMGFCBnZWOKRgBvffD19/jd100xfl3VyciWIJm56Z2jh6TkSqmHR/kL/xRvjZLsaJyK1awemnZ2BHS5bAhRfCaafBGWeE+wA33VTuTcaZKEYDl5jZ44TB7O+jMzpFpIp57DGYPj18mKZDu3bhQ7pfv/RsPyu4w7BhcPXVsG4dHF95ly1JW6Iws5GEQnXbWbgq2I2EQmG4+wOEGjqdCGf+/kS4DoCIVFGtWmWoWyYfffYZnHcejB8PRxwBQ4fCrpVX8ipticLde5axvPDCNSKSx1LpVkpna6JK+OgjmDo1HOy+fcPYRCXKicFsEYlPRccPUhkfyFj/fT6ZORM++ADOPhtOPBHmz4eG6al/qEQhEqNsmI1TlooOBFeJ8YFM+vlnuPnmcNthB+jeHWrXTluSACUKkYwoLSFkw2ycsuiDPou8+y706QOzZsGZZ8Jdd4UkkWZKFCIZUNqsHn0IS8qWLIHDDgutiBdfrNRZTWVRohCpBGV1IRUmCc3qkc32ySew++7QqBE88QQcdRTUT/XKsJVDZcZFKmjIEDj//KJupJJosFY223ffhabmnnvCxInhuZNOyniSALUoRMqtsBVRmCAGD1YXklSS0aPDGdVLl8I118Dvfx9rOEoUIuVUOO6gcQapVH37woMPwj77wPPPQ0FB3BEpUYhsrsKWhMYdpNIkFvErKICdd4Zrr4WaNeONK6JEIbKZEpOExh2kwhYtggsugB494Kyzwv0so0QhUg5qSUiFbdwYBrauvRY2bAgD1VlKiUJEJNM+/TSMRUycCEcfHfozmzWLO6pSKVGIiGTa7Nnw4YcwfDj07l3pRfwqmxKFiEgmzJgRBrd69YKuXUMRv222iTuqlChRSJVV3oJ8Koktm2XtWhg4EP7xD9hxx3Dludq1cyZJgBKF5IHyfuCXtyCfZjtJyt55JxTxmzMnlAO/886MFPGrbEoUkvPKexlNnSgnabVkSfgj++1vYcwY6Ngx7ojKTYlCckpJrQed+CZZZc4c2GuvUMTvySdDEb969eKOqkKUKCQrbc71G9QVJFlhxQro3x/+858w7fWww8KV5/KAEoVkJV2/QXLKc8/BRRfBsmXwxz/GXsSvsilRSNZSd5LkhHPPDa2IVq3gpZegdeu4I6p0ShQiIpsrsYjfgQdC8+Zw9dVQo0a8caWJEoVkxOZOYdW5CpK1vvgiXKnq9NPDlNcq0A+qRCFpUTwxbO45CxqglqyzcSPcfz9cd11oUZx6atwRZYwShVSKshKDBqElp82dG4r4TZoExx4bqr7uskvcUWWMEoVUiuKzlJQYJK/MnQuzZsGIEaG7KcuL+FU2JQqpNJqlJHll2rTw7eecc+CEE0IRv623jjuqWChRSErKGozW4LPkjTVrYMAAuO22cHZ1z56hPlMVTRIA1eIOQHJDYddSaTT4LHnhrbfCH/Mtt4QupunTc7KIX2VTi0JSpq4lyWtLlsARR4RWxNixYdBaALUoJAVDhhTNYhLJO7Nnh5+NGsEzz8BHHylJFKNEIWUqHJtQ15LklW+/DZchbdkyFPED6NIFttoq1rCykbqeJCXt2mmqq+SRZ56Biy+G5cvh+uvhgAPijiirKVHIrxSf4aQZTZJXeveGhx4KxfteeUV/3ClQopBfKX7ynGY0Sc5LLOJ38MHhwkL9+8MW+ghMRVqPkpl1AO4BqgPD3P0fxZY3BR4Cto7Wuc7dx6QzJildYUtCV4yTvLJgQeg3PfNM6NVLfajlkLZEYWbVgUHAMcBi4H0zG+3usxNWuwF40t3vN7MWwBhgl3TFJJtKVp9JLQjJeRs2wKBB4UJC1arBGWfEHVHOSmeL4gBgnrvPBzCzx4GuQGKicKB+dL8B8GUa45FiVJ9J8tacOdCnD7zzDnTsCA88AE2bxh1VzkpnomgELEp4vBhoW2ydm4BXzexSoC5wdEkbMrN+QD+ApvplVyp1MUlemjcvFPJ7+OHQkqhiRfwqW9znUfQERrh7Y6AT8LCZ/Somdx/i7gXuXrD99ttnPMh8M2QItG+fvCSHSM6ZOhWGDw/3u3QJYxNnnqkkUQnSmSiWAE0SHjeOnkvUB3gSwN3fAWoD26UxJmHTLieNRUjOW706XEyobVv4299CUT+A+vWTv05Sls5E8T7Q3MyamVlNoAcwutg6C4GjAMxsL0KiWJbGmCRS2OWk8QjJaRMnwn77wa23hvMjpk1TEb80SNsYhbuvN7NLgLGEqa/D3X2WmQ0Aprj7aKA/MNTMriQMbPd2L5zwLCKSxJIlcNRR0KQJvPZauC9pkdbzKKJzIsYUe+4vCfdnA4ekMwYpUvw8CZGc9NFHsM8+oYjfc8+Fiq9168YdVV6LezBbMqBw8Pr888O5EhqbkJz0zTdw1lmw775FRfw6d1aSyACdv14FFLYidJ6E5CR3eOopuOQSWLECbrwxDFxLxihRVBE6X0JyVq9e4XyIggJ4/fXQ7SQZpUQhItknsYhfu3ahu+mKK1TELyYao8hjOrFOctL8+XD00TBiRHjcpw9cfbWSRIyUKPKQBq8lJ23YAHffHbqW3n8/FPKTrKAUnYc0eC05Z/ZsOPdcePddOP74UMSvceO4o5KIEkWe0uC15JQFC+Czz8K3nB49VJ8pyyhRiEg83n8/NH3POy+0IubPh3r14o5KSqBOQBHJrJ9+CoPTBx4It9xSVMRPSSJrKVGISOZMmBCmuv7zn6EloSJ+OUFdTzms+KVMC6mWk2SlxYvhmGNg551h3LhQo0lygloUOaxwdlNxmg4rWWXGjPCzcWN4/nn48EMliRyjFkUOKd6CKGw5aHaTZKVly+Dyy2HkyPBH2q4ddOoUd1RSDkoUOaAwQbzxRnjcrl34qZaDZCV3ePxxuOwy+P57+Otf4aCD4o5KKkCJIouVlCB0Ap1kvbPOgkcfDRVeH3wQWraMOyKpoJQThZlt6e4/pTMY2ZTOsJacsXFjOEnOLIw/tGkTWhTVq8cdmVSCMhOFmR0MDAO2Apqa2X7A+e5+UbqDE41BSA6YNy9MdT3rrFCGo0+fuCOSSpbKrKe7gOOA5QDuPgM4PJ1BiUgOWL8e7rgjFPGbNg1q1ow7IkmTlLqe3H2RbVp7ZUN6whGRnDBzJpxzDkyZAl27wn33wU47xR2VpEkqiWJR1P3kZlYDuByYk96wRCSrLVwIX3wRZjd1764ifnkula6nC4CLgUbAEqAVoPGJNNIFhyQrvftu+OOEcD7E/Plw2mlKElVAKoliD3c/w913cPffuPuZwF7pDqwqK5ztpPMkJCv8+CNcdVU4F+K222Dt2vD8VlvFG5dkTCqJ4l8pPieVYMiQoqvSTZigKbESs3HjQhG/u+6CCy6ADz6AWrXijkoyrNQxCjM7CDgY2N7MrkpYVB/Q5OhKVvzkOrUkJHaLF8Nxx0GzZuEP83BNdqyqkg1m1yScO7EFkFgo/gfglHQGVRXp5DrJGtOmwf77hyJ+L7wQ/ijr1Ik7KolRqYnC3d8A3jCzEe7+RQZjqlIKWxIq8Cex+9//wtnUTz5ZVMSvQ4e4o5IskMr02J/M7HagJfDLFUbc/ci0RVWFaOBaYuceajNdfjmsWgUDB8LBB8cdlWSRVBLFo8ATQGfCVNlewLJ0BpVPSru4UCG1JCR2p58ezoc46KBQxG8vTWqUTaUy66mhuz8IrHP3N9z9XECtiRSVdnGhQmpJSCw2bgwtCYBjj4V77oE331SSkBKl0qJYF/38ysyOB74Etk1fSLlNFxeSrPfJJ6GI39lnhwJ+55wTd0SS5VJJFAPNrAHQn3D+RH3girRGlQNK61LSxYUka61fD3feCTfeCLVrayaTpKzMROHuL0Z3vweOADCzQ9IZVC5IHIROpOmtkpU+/DCUAJ86FU46CQYNgh13jDsqyRHJTrirDnQn1Hh6xd1nmlln4E9AHWD/zISYvdSlJDlj8WJYtAieegq6dVN9JtksyQazHwT6Ag2Be83sEeAO4DZ3TylJmFkHM5trZvPM7LpS1uluZrPNbJaZJZkfJCKb5e234YEHwv3CIn6nnKIkIZstWddTAbCvu280s9rAUmBXd1+eyoajFskg4BhgMfC+mY1299kJ6zQH/ggc4u4rzOw35X0jIhJZtQquvx7+9S/YddcwWF2rFtStG3dkkqOStSh+dveNAO6+BpifapKIHADMc/f57v4z8DjQtdg65wGD3H1FtJ+vN2P7IlLcq6/C3nuHJHHxxSriJ5UiWYtiTzP7MLpvwK7RYwPc3fctY9uNgEUJjxcDbYutszuAmb1FKDR4k7u/UnxDZtYP6AfQtGnTMnYrUkUtWgTHHx9aERMnwqGHxh2R5IlkiSITZ95sATQH2gONgYlmto+7f5e4krsPAYYAFBQUeAbiEskdU6dCmzbQpAmMGQOHHRamv4pUklK7ntz9i2S3FLa9BGiS8Lhx9FyixcBod1/n7guATwiJI2vp6nOSNZYuhVNPhYKCohN4jjlGSUIqXSolPMrrfaC5mTUzs5pAD2B0sXVGEVoTmNl2hK6o+WmMqdwKE8T55xddWEgn0kks3OGhh6BFi1AG/OabVcRP0iqVM7PLxd3Xm9klwFjC+MNwd59lZgOAKe4+Olp2rJnNBjYA12zmgHnG6HoRkjV69AilwA85BIYNgz33jDsiyXPmXnaXv5nVAZq6+9z0h5RcQUGBT5kyJeP7bd8+/NQJdhKLjRvD+Q9moTWxciVcdBFUS2engOQTM5vq7gXleW2Zf2Vm1gWYDrwSPW5lZsW7kEQkXT7+OFyG9MEHw+NeveCSS5QkJGNS+Uu7iXBOxHcA7j4daJbGmEQEYN26MP6w334wezZstVXcEUkVlUqiWOfu3xd7rspMUdUsJ4nF9OlwwAHhDOsTTgiJokePuKOSKiqVwexZZnY6UD0quXEZ8HZ6w8oeulSpxGLp0nB75hk4+eS4o5EqLpVEcSlwPbAWeIwwU2lgOoPKNqoSKxkxaVIoB37RRdChA3z2GWy5ZdxRiaTU9bSnu1/v7r+PbjdEtZ9EpDKsXBkGpw87DO6+G9auDc8rSUiWSCVR/NPM5pjZ38xs77RHJFKVjB0bivjddx9cfrmK+ElWKjNRuPsRhCvbLQMGm9lHZnZD2iMTyXeLFkHnzqHlMGlSaE1oZpNkoZQmYrv7Une/F7iAcE7FX9IalUi+cof33gv3mzSBl1+GadNUgkOyWion3O1lZjeZ2UfAvwgznhqnPTKRfPPVV+EypG3bFhXxO/poFfGTrJfKrKfhwBPAce7+ZZrjEck/7jBiBFx1FaxZA7feGuo0ieSIMhOFux+UiUBE8lb37vD002FW07BhsPvucUcksllKTRRm9qS7d4+6nBLPxE71CnciVdeGDaGAX7Vq0KULHHlkqFGv+kySg5K1KC6PfnbORCAieWPOHOjTB845B847D84+O+6IRCok2RXuvoruXlTC1e0uykx4Ijlk3ToYODCcyj93LjRoEHdEIpUilXbwMSU817GyAxHJadOmhUuS/vnPcNJJoVXRvXvcUYlUimRjFBcSWg6/M7MPExbVA95Kd2AiOeV//4NvvoFRo6Br17ijEalUycYoHgNeBm4Brkt4fqW7f5vWqERywcSJ8NFHcPHFoYjfvHlQp07cUYlUumRdT+7unwMXAysTbpjZtukPTSRL/fBDqPDarh3ce29RET8lCclTZbUoOgNTCdNjLWGZA79LY1wi2WnMmDDN9csvwwl0AwaoiJ/kvVIThbt3jn7qsqciEIr4de0Ke+wRTqBr2zbuiEQyIpVaT4eYWd3o/plmdqeZNU1/aCJZwB0mTw73mzSBV18NpcCVJKQKSWV67P3AT2a2H9Af+Ax4OK1RiWSDL7+EE0+Egw4qKuJ3xBFQs2a8cYlkWCqJYr27O9AV+Le7DyJMkRXJT+6hJlOLFqEFcccdKuInVVoq1WNXmtkfgbOAw8ysGlAjvWGJxOiUU+DZZ8OspmHDYLfd4o5IJFaptChOA9YC57r7UsK1KG5Pa1RZYMgQaN8epk+POxLJiA0bYOPGcP/EE+GBB2DcOCUJEVK7FOpS4FGggZl1Bta4+/+lPbKYPfZYSBKtWsHpp8cdjaTVzJmha+nBB8Pjs85SpVeRBKnMeuoOvAecCnQH3jWzU9IdWFwSWxKtWsGECdCvX9xRSVr8/DP89a/QujV89hlss03cEYlkpVTGKK4Hfu/uXwOY2fbAa8DT6Qws04YMCa2Iwskt7dqpJZHXpk6F3r1Da+L00+Huu2H77eOOSiQrpZIoqhUmichyUhvbyCmFXU2FCUKtiDy3fDl89x288AJ01iVXRJJJJVG8YmZjgZHR49OAMekLKT6FXU2Sp8aPD0X8LrsMjj0WPv0UateOOyqRrJfKYPY1wGBg3+g2xN2vTXdgIpXm++/D4PSRR8L99xcV8VOSEElJsutRNAfuAHYFPgKudvclmQpMpFK88AJccAEsXQpXXx0Gr1XET2SzJGtRDAdeBLoRKsj+KyMRiVSWRYugWzdo2DDUa7r9dthyy7ijEsk5ycYo6rn70Oj+XDP7IBMBiVSIO7zzDhx8cFERv4MPVn0mkQpI1qKobWb7m1lrM2sN1Cn2uExm1sHM5prZPDO7Lsl63czMzaxgc9+AyC8WL4YTTggnzxXOc27fXklCpIKStSi+Au5MeLw04bEDRybbsJlVBwYBxwCLgffNbLS7zy62Xj3gcuDdzQu9chSeP1F4gp3koI0bYehQuOYaWL8e7rwTDj007qhE8kayCxcdUcFtHwDMc/f5AGb2OKEC7exi6/0NuBW4poL7KxeV6sgD3brBqFFhVtPQofA7XXxRpDKlch5FeTUCFiU8XgxscrWXqAuribu/ZGalJgoz6wf0A2jatPKvmaTzJ3LQ+vWhFlO1aiFRHH889OkDZmW/VkQ2S2xnWEflyu8kXAwpKXcf4u4F7l6wvcosyIcfhosJDY3mWpx5JvTtqyQhkibpTBRLgCYJjxtHzxWqB+wNTDCzz4EDgdGZGNAuLPynMuI5Zu1auPFGaNMGvvhCtZlEMiSV6rEWXSv7L9HjpmZ2QArbfh9obmbNzKwm0AMYXbjQ3b939+3cfRd33wWYDJzg7lPK9U42Q+G4BGhsIme8/36o8jpgAPTsCXPmwMknxx2VSJWQyhjFfcBGwiynAcBK4Bng98le5O7rzewSYCxQHRju7rPMbAAwxd1HJ3t9umlcIsesWAGrVsGYMdCxY9zRiFQpqSSKtu7e2symAbj7iqiFUCZ3H0OxAoLu/pdS1m2fyjalChk3LhTxu/zyUMTvk09UfkMkBqmMUayLzolw+OV6FBvTGpVUbd99B+edB0cdBYMHFxXxU5IQiUUqieJe4DngN2b2d2AScHNao5Kq6/nnoUULGD4c/vCHcIEhJQiRWJXZ9eTuj5rZVOAowIAT3X1O2iNLA52FneUWLoRTT4W99oLRo6FAFV1EskGZicLMmgI/AS8kPufuC9MZWDroLOws5A6TJsFhh0HTpvDaa3DggarPJJJFUhnMfokwPmFAbaAZMBdomca40kaznbLIwoXhWhEvvxx+Ke3aweGHxx2ViBSTStfTPomPo7IbF6UtIsl/GzfCAw/AtdeGFsW996qIn0gW2+xaT+7+gZm1LXtNkVKcfHIYtD7mmDBwtMsucUckIkmkMkZxVcLDakBr4Mu0RST5KbGI32mnQdeu0Lu36jOJ5IBUpsfWS7jVIn4IvZoAABS1SURBVIxZdE1nUJJnZsyAtm1D6wFCCY5zzlGSEMkRSVsU0Yl29dz96gzFI/lkzRoYOBBuvRW23RZ++9u4IxKRcig1UZjZFlG9pkMyGZDkiffeg1694OOPw8877wzJQkRyTrIWxXuE8YjpZjYaeAr4sXChuz+b5tgqjU60i8EPP8Dq1fDKK3DccXFHIyIVkMqsp9rAckL12MLzKRzImUShE+0y5NVXYdYsuPJKOPpomDtX5TdE8kCyRPGbaMbTTIoSRCFPa1RpoBPt0mjFCrjqKhgxAlq2hIsuCglCSUIkLySb9VQd2Cq61Uu4X3gTgWefDUX8Hn4Y/vhHmDJFCUIkzyRrUXzl7gMyFonknoULoUcP2HvvcEGh/fePOyIRSYNkLQpNcpdfc4c33gj3mzYNFxd6910lCZE8lixRHJWxKCQ3fPFFuAxp+/ZFyeLQQ6FGjVjDEpH0KjVRuPu3mQxEstjGjfDvf4eB6kmT4F//CmXBRaRKSKWER84aMiR8+Z0+Pe5IctyJJ8Kll4bWw6xZcMkloWaTiFQJm109Npfo/IkKWLcOqlcPCaFnTzjlFDjrLNVnEqmC8jpRgM6fKJcPPoA+feC888I5ET17xh2RiMRI/QdSZPXqcC7EAQfA0qXQpEncEYlIFsj7FoWkaPLkULzvk0/g3HPhjjtgm23ijkpEsoAShQQ//hjGJf7731CnSUQkokRRlb3ySpjF1L8/HHVUKAles2bcUYlIltEYRVW0fHnoZurYER56CH7+OTyvJCEiJVCiqErc4emnQxG/xx6DG26A999XghCRpNT1VJUsXBhOKNl333DtiP32izsiEckBalHkO/dQuA9g553DSSWTJytJiEjKlCjy2YIFcOyxYaC6sIjfwQfDFmpIikjqlCjy0YYNcM894ToR774L99+vIn4iUm76apmPunaFl16CTp3ggQd0hrWIVEhetiiqZNXYdetCOXAIxfseeQRefFFJQkQqLK2Jwsw6mNlcM5tnZteVsPwqM5ttZh+a2etmtnNl7LfKVY2dMgUKCkIXE8Bpp8EZZ6jSq4hUirQlCjOrDgwCOgItgJ5m1qLYatOAAnffF3gauK2y9l9YNbZfv8raYhZavRquvRbatoVly8KsJhGRSpbOFsUBwDx3n+/uPwOPA10TV3D38e7+U/RwMtA4jfHkl3feCVNcb7stFPGbPRs6d447KhHJQ+lMFI2ARQmPF0fPlaYP8HJJC8ysn5lNMbMpy5YtK3UDVWpsYvXqMCbx2mswdChsvXXcEYlInsqKWU9mdiZQALQrabm7DwGGABQUFHhp28n7sYkxY0IRv2uugSOPhDlzoEaNuKMSkTyXzkSxBEicctM4em4TZnY0cD3Qzt3XVnSneXlFu2++gSuugEcfDd1Nl18e6jMpSYhIBqSz6+l9oLmZNTOzmkAPYHTiCma2PzAYOMHdvy7vjvK2y8kdHn8c9toLnnwSbrwR3ntPRfxEJKPS1qJw9/VmdgkwFqgODHf3WWY2AJji7qOB24GtgKcsTOVc6O4nbO6+8rbLaeHCUA58v/3gwQdhn33ijkhEqqC0jlG4+xhgTLHn/pJwv9IupZY3XU7u8Prr4SpzO+8cajT9/vdQvXrckYlIFZWXZ2bnrM8+CwX8jjmmqIjfgQcqSYhIrJQossGGDXDnnaFraepUGDxYRfxEJGtkxfTYKq9LF3j55XDC3P33Q2Oddygi2UOJIi4//xyuC1GtGvTuHQr59eih+kwiknXU9RSH996DNm3gvvvC4+7doWdPJQkRyUpKFJn000/Qvz8cdBCsWAG77hp3RCIiZcrpRJFTJ9pNmhQGq++8E847L5Ti6Ngx7qhERMqUk2MUQ4aEk+wKZ5C2a5cDJ9qtWxemuY4fH7KbiEiOMPdSa+xlpXr1CnzVqilAUYLI2mtOvPBCKNz3hz+Ex+vXhwFsEZEMM7Op7l5QntfmXNfT6tUhQQwenMUXJlq2LGSwE06AkSPDDCdQkhCRnJRzn1x16mRxqQ73kBguuwx++AEGDAhXoFMRPxHJYTmXKLLawoVwzjmw//6hiF/LlnFHJCJSYTnX9ZR1Nm6EsWPD/Z13hjffhLfeUpIQkbyhRFERn34arjTXoQNMnBieO+AAFfETkbyiRFEe69fD7bfDvvuGkzgefFBF/EQkb2mMojw6dw7dTV27hjIcO+0Ud0QiWWndunUsXryYNWvWxB1KlVG7dm0aN25MjUq8VHJOnkexcuWUzO947dpwjepq1eDpp8PYxKmnqj6TSBILFiygXr16NGzYENP/Stq5O8uXL2flypU0a9Zsk2VV6jyKWEyeDK1bw6BB4fEpp4RCfvrDF0lqzZo1ShIZZGY0bNiw0ltwShTJ/PgjXHklHHwwrFwJzZvHHZFIzlGSyKx0HG+NUZTmzTehVy9YsAAuughuuQXq1487KhGRjFOLojTr14cxiTfeCF1OShIiOWvUqFGYGR9//PEvz02YMIHOnTtvsl7v3r15+umngTAQf91119G8eXNat27NQQcdxMsvv1zhWG655RZ222039thjD8YWnoNVzOuvv07r1q1p1aoVhx56KPPmzQNgxIgRbL/99rRq1YpWrVoxbNiwCseTCiWKRKNGhZYDwBFHhFLghx8eb0wiUmEjR47k0EMPZeTIkSm/5s9//jNfffUVM2fO5IMPPmDUqFGsXLmyQnHMnj2bxx9/nFmzZvHKK69w0UUXsWHDhl+td+GFF/Loo48yffp0Tj/9dAYOHPjLstNOO43p06czffp0+vbtW6F4UqWuJ4D//Q8uvRSeeioMWvfvH+ozqYifSKW54orKv3ZMq1Zw993J11m1ahWTJk1i/PjxdOnShb/+9a9lbvenn35i6NChLFiwgFq1agGwww470L179wrF+/zzz9OjRw9q1apFs2bN2G233Xjvvfc46KCDNlnPzPjhhx8A+P7779kp5in4VfuT0B0eeST8Ba9aBX//O1xzTehyEpG88Pzzz9OhQwd23313GjZsyNSpU2nTpk3S18ybN4+mTZtSP4Uu5yuvvJLx48f/6vkePXpw3XXXbfLckiVLOPDAA3953LhxY5YsWfKr1w4bNoxOnTpRp04d6tevz+TJk39Z9swzzzBx4kR233137rrrLpo0aVJmjBVVtRPFwoXQty8UFISzq/fcM+6IRPJWWd/802XkyJFcfvnlQPjwHjlyJG3atCl1dtDmzhq66667KhxjSdscM2YMbdu25fbbb+eqq65i2LBhdOnShZ49e1KrVi0GDx5Mr169GDduXKXvv7iqlygKi/h17BiK+L31Vqj2qvpMInnn22+/Zdy4cXz00UeYGRs2bMDMuP3222nYsCErVqz41frbbbcdu+22GwsXLuSHH34os1WxOS2KRo0asWjRol8eL168mEaNGm2yzrJly5gxYwZt27YFwphEhw4dAGjYsOEv6/Xt25c/FF4ULd3cPaduW23Vxstt7lz3ww5zB/cJE8q/HRFJyezZs2Pd/+DBg71fv36bPHf44Yf7G2+84WvWrPFddtnllxg///xzb9q0qX/33Xfu7n7NNdd47969fe3ate7u/vXXX/uTTz5ZoXhmzpzp++67r69Zs8bnz5/vzZo18/Xr12+yzrp167xhw4Y+d+5cd3cfNmyYn3zyye7u/uWXX/6y3rPPPutt27YtcT8lHXdgipfzc7dqtCjWr4d//hNuvDFc+eg//9FsJpEqYOTIkVx77bWbPNetWzdGjhzJ4YcfziOPPMI555zDmjVrqFGjBsOGDaNBgwYADBw4kBtuuIEWLVpQu3Zt6taty4ABAyoUT8uWLenevTstWrRgiy22YNCgQVSPejM6derEsGHD2GmnnRg6dCjdunWjWrVqbLPNNgwfPhyAe++9l9GjR7PFFluw7bbbMmLEiArFk6qqUevpuOPg1Vfh5JPDORG//W16ghORTcyZM4e99tor7jCqnJKOe0VqPeVvi2LNmjB7qXr1cGHtfv2gW7e4oxIRyTn5ecLdW2+FCdaFRfy6dVOSEBEpp/xKFKtWwWWXhYsIrVkDavKKxC7XurdzXTqOd/4kijfegL33hn//Gy65BGbOhGOOiTsqkSqtdu3aLF++XMkiQzy6HkXt2rUrdbv5NUax5Zah6ushh8QdiYgQzjxevHgxy5YtizuUKqPwCneVKbdnPT37LHz8MfzpT+Hxhg06cU5EpARZe4U7M+tgZnPNbJ6ZXVfC8lpm9kS0/F0z2yWlDS9dGq4y160bPPcc/PxzeF5JQkSk0qUtUZhZdWAQ0BFoAfQ0sxbFVusDrHD33YC7gFvL2m6DdcvDIPWLL4aS4G+/HSq9iohIWqSzRXEAMM/d57v7z8DjQNdi63QFHoruPw0cZWVU5Nph7Rdh0HrGDLjuOlV6FRFJs3QOZjcCFiU8Xgy0LW0dd19vZt8DDYFvElcys35Av+jhWps0aaYqvQKwHcWOVRWmY1FEx6KIjkWRPcr7wpyY9eTuQ4AhAGY2pbwDMvlGx6KIjkURHYsiOhZFzGwzax8VSWfX0xIg8YoajaPnSlzHzLYAGgDL0xiTiIhspnQmiveB5mbWzMxqAj2A0cXWGQ30iu6fAozzXJuvKyKS59LW9RSNOVwCjAWqA8PdfZaZDSDURR8NPAg8bGbzgG8JyaQsQ9IVcw7SsSiiY1FEx6KIjkWRch+LnDvhTkREMit/aj2JiEhaKFGIiEhSWZso0lb+IwelcCyuMrPZZvahmb1uZjvHEWcmlHUsEtbrZmZuZnk7NTKVY2Fm3aO/jVlm9limY8yUFP5HmprZeDObFv2fdIojznQzs+Fm9rWZzSxluZnZvdFx+tDMWqe04fJebDudN8Lg92fA74CawAygRbF1LgIeiO73AJ6IO+4Yj8URwJbR/Qur8rGI1qsHTAQmAwVxxx3j30VzYBqwTfT4N3HHHeOxGAJcGN1vAXwed9xpOhaHA62BmaUs7wS8DBhwIPBuKtvN1hZFWsp/5Kgyj4W7j3f3n6KHkwnnrOSjVP4uAP5GqBu2JpPBZVgqx+I8YJC7rwBw968zHGOmpHIsHKgf3W8AfJnB+DLG3ScSZpCWpivwfx5MBrY2sx3L2m62JoqSyn80Km0dd18PFJb/yDepHItEfQjfGPJRmcciako3cfeXMhlYDFL5u9gd2N3M3jKzyWbWIWPRZVYqx+Im4EwzWwyMAS7NTGhZZ3M/T4AcKeEhqTGzM4ECoF3cscTBzKoBdwK9Yw4lW2xB6H5qT2hlTjSzfdz9u1ijikdPYIS7/9PMDiKcv7W3u2+MO7BckK0tCpX/KJLKscDMjgauB05w97UZii3TyjoW9YC9gQlm9jmhD3Z0ng5op/J3sRgY7e7r3H0B8AkhceSbVI5FH+BJAHd/B6hNKBhY1aT0eVJctiYKlf8oUuaxMLP9gcGEJJGv/dBQxrFw9+/dfTt338XddyGM15zg7uUuhpbFUvkfGUVoTWBm2xG6ouZnMsgMSeVYLASOAjCzvQiJoipen3U0cHY0++lA4Ht3/6qsF2Vl15Onr/xHzknxWNwObAU8FY3nL3T3E2ILOk1SPBZVQorHYixwrJnNBjYA17h73rW6UzwW/YGhZnYlYWC7dz5+sTSzkYQvB9tF4zE3AjUA3P0BwvhMJ2Ae8BNwTkrbzcNjJSIilShbu55ERCRLKFGIiEhSShQiIpKUEoWIiCSlRCEiIkkpUUhWMrMNZjY94bZLknVXVcL+RpjZgmhfH0Rn727uNoaZWYvo/p+KLXu7ojFG2yk8LjPN7AUz27qM9Vvla6VUyRxNj5WsZGar3H2ryl43yTZGAC+6+9Nmdixwh7vvW4HtVTimsrZrZg8Bn7j735Os35tQQfeSyo5Fqg61KCQnmNlW0bU2PjCzj8zsV1VjzWxHM5uY8I37sOj5Y83snei1T5lZWR/gE4HdotdeFW1rppldET1X18xeMrMZ0fOnRc9PMLMCM/sHUCeK49Fo2aro5+NmdnxCzCPM7BQzq25mt5vZ+9F1As5P4bC8Q1TQzcwOiN7jNDN728z2iM5SHgCcFsVyWhT7cDN7L1q3pOq7IpuKu366brqVdCOcSTw9uj1HqCJQP1q2HeHM0sIW8aroZ3/g+uh+dULtp+0IH/x1o+evBf5Swv5GAKdE908F3gXaAB8BdQlnvs8C9ge6AUMTXtsg+jmB6PoXhTElrFMY40nAQ9H9moRKnnWAfsAN0fO1gClAsxLiXJXw/p4COkSP6wNbRPePBp6J7vcG/p3w+puBM6P7WxPqP9WN+/etW3bfsrKEhwiw2t1bFT4wsxrAzWZ2OLCR8E16B2BpwmveB4ZH645y9+lm1o5woZq3ovImNQnfxEtyu5ndQKgB1IdQG+g5d/8xiuFZ4DDgFeCfZnYrobvqzc14Xy8D95hZLaADMNHdV0fdXfua2SnReg0IBfwWFHt9HTObHr3/OcB/E9Z/yMyaE0pU1Chl/8cCJ5jZ1dHj2kDTaFsiJVKikFxxBrA90Mbd11moDls7cQV3nxglkuOBEWZ2J7AC+K+790xhH9e4+9OFD8zsqJJWcvdPLFz3ohMw0Mxed/cBqbwJd19jZhOA44DTCBfZgXDFsUvdfWwZm1jt7q3MbEtCbaOLgXsJF2sa7+4nRQP/E0p5vQHd3H1uKvGKgMYoJHc0AL6OksQRwK+uC27hWuH/c/ehwDDCJSEnA4eYWeGYQ10z2z3Ffb4JnGhmW5pZXUK30ZtmthPwk7s/QijIWNJ1h9dFLZuSPEEoxlbYOoHwoX9h4WvMbPdonyXycEXDy4D+VlRmv7BcdO+EVVcSuuAKjQUutah5ZaHysEhSShSSKx4FCszsI+Bs4OMS1mkPzDCzaYRv6/e4+zLCB+dIM/uQ0O20Zyo7dPcPCGMX7xHGLIa5+zRgH+C9qAvoRmBgCS8fAnxYOJhdzKuEi0u95uHSnRAS22zgAzObSSgbn7TFH8XyIeGiPLcBt0TvPfF144EWhYPZhJZHjSi2WdFjkaQ0PVZERJJSi0JERJJSohARkaSUKEREJCklChERSUqJQkREklKiEBGRpJQoREQkqf8Hggy/dUz7FQUAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lEPPYHa62JXF"
      },
      "source": [
        "# D - Fine-tuning BERT"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bYJRzWI73eBJ"
      },
      "source": [
        "## 1. Install the Hugging Face Library"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yxv-EJ2j31Iv"
      },
      "source": [
        "The transformer library of Hugging Face contains PyTorch implementation of state-of-the-art NLP models including BERT (from Google), GPT (from OpenAI) ... and pre-trained model weights."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uFiv8WGl4p40",
        "outputId": "1c9dc470-be0c-4077-fdf4-16dfc72f15e8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 683
        }
      },
      "source": [
        "!pip install transformers==2.8.0"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a3/78/92cedda05552398352ed9784908b834ee32a0bd071a9b32de287327370b7/transformers-2.8.0-py3-none-any.whl (563kB)\n",
            "\r\u001b[K     |▋                               | 10kB 28.7MB/s eta 0:00:01\r\u001b[K     |█▏                              | 20kB 6.5MB/s eta 0:00:01\r\u001b[K     |█▊                              | 30kB 7.8MB/s eta 0:00:01\r\u001b[K     |██▎                             | 40kB 5.9MB/s eta 0:00:01\r\u001b[K     |███                             | 51kB 6.5MB/s eta 0:00:01\r\u001b[K     |███▌                            | 61kB 7.7MB/s eta 0:00:01\r\u001b[K     |████                            | 71kB 7.5MB/s eta 0:00:01\r\u001b[K     |████▋                           | 81kB 7.1MB/s eta 0:00:01\r\u001b[K     |█████▎                          | 92kB 7.9MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 102kB 8.1MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 112kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████                         | 122kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 133kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 143kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████▊                       | 153kB 8.1MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 163kB 8.1MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 174kB 8.1MB/s eta 0:00:01\r\u001b[K     |██████████▌                     | 184kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████                     | 194kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 204kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 215kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 225kB 8.1MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 235kB 8.1MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 245kB 8.1MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 256kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 266kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 276kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 286kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 296kB 8.1MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 307kB 8.1MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 317kB 8.1MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 327kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 337kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 348kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 358kB 8.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 368kB 8.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 378kB 8.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 389kB 8.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 399kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████▎        | 409kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 419kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 430kB 8.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 440kB 8.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 450kB 8.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 460kB 8.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 471kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 481kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 491kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 501kB 8.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 512kB 8.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 522kB 8.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 532kB 8.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 542kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 552kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 563kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 573kB 8.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.12.38)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/8d/4d/7ef0a02bb9ce669b90d576e57ca064191aa632d3bfb6a35ad2a7e63d7497/sacremoses-0.0.40.tar.gz (868kB)\n",
            "\u001b[K     |████████████████████████████████| 870kB 20.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
            "\u001b[K     |████████████████████████████████| 1.0MB 32.8MB/s \n",
            "\u001b[?25hCollecting tokenizers==0.5.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/3f/73c881ea4723e43c1e9acf317cf407fab3a278daab3a69c98dcac511c04f/tokenizers-0.5.2-cp36-cp36m-manylinux1_x86_64.whl (3.7MB)\n",
            "\u001b[K     |████████████████████████████████| 3.7MB 54.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.21.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.38.0)\n",
            "Requirement already satisfied: botocore<1.16.0,>=1.15.38 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (1.15.38)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.3.3)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.9.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.1)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.4.5.1)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.8)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.38->boto3->transformers) (2.8.1)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.38->boto3->transformers) (0.15.2)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.40-cp36-none-any.whl size=893351 sha256=561d70bbfb42d1cff03992de3b62048b6d70d8696c8f59a0956ac41130b95599\n",
            "  Stored in directory: /root/.cache/pip/wheels/ba/44/6e/939fb8625572dca96344566c5167bc4f8ab079294509898141\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, sentencepiece, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.40 sentencepiece-0.1.85 tokenizers-0.5.2 transformers-2.8.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D4sXctSh4sq0"
      },
      "source": [
        "## 2. Tokenization and Input Formatting"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ygbZpK6qbIYE"
      },
      "source": [
        "Before tokenizing our text, we will perform some slight processing on our text including removing entity mentions (eg. @united) and some special character. The level of processing here is much less than in previous approachs because BERT was trained with the entire sentences."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4L_Rc7l4bgzJ"
      },
      "source": [
        "def text_preprocessing(text):\n",
        "    \"\"\"\n",
        "    - Remove entity mentions (eg. '@united')\n",
        "    - Correct errors (eg. '&amp;' to '&')\n",
        "    @param    text (str): a string to be processed.\n",
        "    @return   text (Str): the processed string.\n",
        "    \"\"\"\n",
        "    # Remove '@name'\n",
        "    text = re.sub(r'(@.*?)[\\s]', ' ', text)\n",
        "\n",
        "    # Replace '&amp;' with '&'\n",
        "    text = re.sub(r'&amp;', '&', text)\n",
        "\n",
        "    # Remove trailing whitespace\n",
        "    text = re.sub(r'\\s+', ' ', text).strip()\n",
        "\n",
        "    return text"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qyYmHR8McE0r",
        "outputId": "a83c2977-7357-4044-d0be-dc365f414b3f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# Print sentence 0\n",
        "print('Original: ', X[0])\n",
        "print('Processed: ', text_preprocessing(X[0]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original:  @united I'm having issues. Yesterday I rebooked for 24 hours after I was supposed to fly, now I can't log on &amp; check in. Can you help?\n",
            "Processed:  I'm having issues. Yesterday I rebooked for 24 hours after I was supposed to fly, now I can't log on & check in. Can you help?\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i3acv6s95YYr"
      },
      "source": [
        "### 2.1. BERT Tokenizer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C1fRHtdU5dEn"
      },
      "source": [
        "In order to apply the pre-trained BERT, we must use the tokenizer provided by the library. This is because (1) the model has a specific, fixed vocabulary and (2) the BERT tokenizer has a particular way of handling out-of-vocabulary words.\n",
        "\n",
        "In addition, we are required to add special tokens to the start and end of each sentence, pad & truncate all sentences to a single constant length, and explicitly specify what are padding tokens with the \"attention mask\".\n",
        "\n",
        "The `encode_plus` method of BERT tokenizer will:\n",
        "\n",
        "(1) split our text into tokens,\n",
        "\n",
        "(2) add the special `[CLS]` and `[SEP]` tokens, and\n",
        "\n",
        "(3) convert these tokens into indexes of the tokenizer vocabulary,\n",
        "\n",
        "(4) pad or truncate sentences to max length, and\n",
        "\n",
        "(5) create attention mask.\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yDAfbCle59tP",
        "outputId": "253338f0-2440-42b3-c4e8-66287d57773a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "961b5b49f3cf4828ba3510d721bd57db",
            "30cb683585664b7a95ddd24de514c103",
            "a65ea33fe937416bbc29cbebdb025f93",
            "7257f203a0de462789458073993d9948",
            "527af73b1cdd4f3ba035daceb641ad78",
            "fdb8ea4ab0814e6da137331241694bdf",
            "8a7c1e4357444b92bb9821a07cd693ae",
            "dc7f6589b9be45c5b89a1a088ff5d375"
          ]
        }
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "\n",
        "# Load the BERT tokenizer\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
        "\n",
        "# Create a function to tokenize a set of texts\n",
        "def preprocessing_for_bert(data):\n",
        "    \"\"\"Perform required preprocessing steps for pretrained BERT.\n",
        "    @param    data (np.array): Array of texts to be processed.\n",
        "    @return   input_ids (torch.Tensor): Tensor of token ids to be fed to a model.\n",
        "    @return   attention_masks (torch.Tensor): Tensor of indices specifying which\n",
        "                  tokens should be attended to by the model.\n",
        "    \"\"\"\n",
        "    # Create empty lists to store outputs\n",
        "    input_ids = []\n",
        "    attention_masks = []\n",
        "\n",
        "    # For every sentence...\n",
        "    for sent in data:\n",
        "        # `encode_plus` will:\n",
        "        #    (1) Tokenize the sentence\n",
        "        #    (2) Add the `[CLS]` and `[SEP]` token to the start and end\n",
        "        #    (3) Truncate/Pad sentence to max length\n",
        "        #    (4) Map tokens to their IDs\n",
        "        #    (5) Create attention mask\n",
        "        #    (6) Return a dictionary of outputs\n",
        "        encoded_sent = tokenizer.encode_plus(\n",
        "            text=text_preprocessing(sent),  # Preprocess sentence\n",
        "            add_special_tokens=True,        # Add `[CLS]` and `[SEP]`\n",
        "            max_length=MAX_LEN,                  # Max length to truncate/pad\n",
        "            pad_to_max_length=True,         # Pad sentence to max length\n",
        "            #return_tensors='pt',           # Return PyTorch tensor\n",
        "            return_attention_mask=True      # Return attention mask\n",
        "            )\n",
        "\n",
        "        # Add the outputs to the lists\n",
        "        input_ids.append(encoded_sent.get('input_ids'))\n",
        "        attention_masks.append(encoded_sent.get('attention_mask'))\n",
        "\n",
        "    # Convert lists to tensors\n",
        "    input_ids = torch.tensor(input_ids)\n",
        "    attention_masks = torch.tensor(attention_masks)\n",
        "\n",
        "    return input_ids, attention_masks"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "961b5b49f3cf4828ba3510d721bd57db",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=231508, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TNE9oASMZ1bN"
      },
      "source": [
        "Before tokenizing, we need to specify the maximum length of our sentences."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hrbvKGNAlMtt",
        "outputId": "23b50620-4a3e-42ac-ece1-e74cd0f626bc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Concatenate train data and test data\n",
        "all_tweets = np.concatenate([data.tweet.values, test_data.tweet.values])\n",
        "\n",
        "# Encode our concatenated data\n",
        "encoded_tweets = [tokenizer.encode(sent, add_special_tokens=True) for sent in all_tweets]\n",
        "\n",
        "# Find the maximum length\n",
        "max_len = max([len(sent) for sent in encoded_tweets])\n",
        "print('Max length: ', max_len)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Max length:  68\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vpdjBB9fmbu2"
      },
      "source": [
        "Now let's tokenize our data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QTlQzTzAfCy7",
        "outputId": "933db5ca-b529-4ec9-cdda-e94851f7dc73",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        }
      },
      "source": [
        "# Specify `MAX_LEN`\n",
        "MAX_LEN = 64\n",
        "\n",
        "# Print sentence 0 and its encoded token ids\n",
        "token_ids = list(preprocessing_for_bert([X[0]])[0].squeeze().numpy())\n",
        "print('Original: ', X[0])\n",
        "print('Token IDs: ', token_ids)\n",
        "\n",
        "# Run function `preprocessing_for_bert` on the train set and the validation set\n",
        "print('Tokenizing data...')\n",
        "train_inputs, train_masks = preprocessing_for_bert(X_train)\n",
        "val_inputs, val_masks = preprocessing_for_bert(X_val)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original:  @united I'm having issues. Yesterday I rebooked for 24 hours after I was supposed to fly, now I can't log on &amp; check in. Can you help?\n",
            "Token IDs:  [101, 1045, 1005, 1049, 2383, 3314, 1012, 7483, 1045, 2128, 8654, 2098, 2005, 2484, 2847, 2044, 1045, 2001, 4011, 2000, 4875, 1010, 2085, 1045, 2064, 1005, 1056, 8833, 2006, 1004, 4638, 1999, 1012, 2064, 2017, 2393, 1029, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
            "Tokenizing data...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aZU8t5VNfvhY"
      },
      "source": [
        "### 2.2. Create PyTorch DataLoader"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aoHdl3gFgMZY"
      },
      "source": [
        "We will create an iterator for our dataset using the torch DataLoader class. This will help save on memory during training and boost the training speed."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xHuYEc61gcGL"
      },
      "source": [
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "# Convert other data types to torch.Tensor\n",
        "train_labels = torch.tensor(y_train)\n",
        "val_labels = torch.tensor(y_val)\n",
        "\n",
        "# For fine-tuning BERT, the authors recommend a batch size of 16 or 32.\n",
        "batch_size = 32\n",
        "\n",
        "# Create the DataLoader for our training set\n",
        "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "# Create the DataLoader for our validation set\n",
        "val_data = TensorDataset(val_inputs, val_masks, val_labels)\n",
        "val_sampler = SequentialSampler(val_data)\n",
        "val_dataloader = DataLoader(val_data, sampler=val_sampler, batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SSRAga-yj17q"
      },
      "source": [
        "## 3. Train Our Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KoOdsDgG8b_Z"
      },
      "source": [
        "### 3.1. Create BertClassifier"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zA_yESCl5nuK"
      },
      "source": [
        "BERT-base consists of 12 transformer layers, each transformer layer takes in a list of token embeddings, and produces the same number of embeddings with the same hidden size (or dimensions) on the output. The output of the final transformer layer of the `[CLS]` token is used as the features of the sequence to feed a classifier.\n",
        "\n",
        "The `transformers` library has the [`BertForSequenceClassification`](https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#bertforsequenceclassification) class which is designed for classification tasks. However, we will create a new class so we can specify our own choice of classifiers.\n",
        "\n",
        "Below we will create a BertClassifier class with a BERT model to extract the last hidden layer of the `[CLS]` token and a single-hidden-layer feed-forward neural network as our classifier."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YK41aBFSj5jK",
        "outputId": "ce7e0b40-5790-47ff-8c85-edda7ed9a6cb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%%time\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from transformers import BertModel\n",
        "\n",
        "# Create the BertClassfier class\n",
        "class BertClassifier(nn.Module):\n",
        "    \"\"\"Bert Model for Classification Tasks.\n",
        "    \"\"\"\n",
        "    def __init__(self, freeze_bert=False):\n",
        "        \"\"\"\n",
        "        @param    bert: a BertModel object\n",
        "        @param    classifier: a torch.nn.Module classifier\n",
        "        @param    freeze_bert (bool): Set `False` to fine-tune the BERT model\n",
        "        \"\"\"\n",
        "        super(BertClassifier, self).__init__()\n",
        "        # Specify hidden size of BERT, hidden size of our classifier, and number of labels\n",
        "        D_in, H, D_out = 768, 50, 2\n",
        "\n",
        "        # Instantiate BERT model\n",
        "        self.bert = BertModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "        # Instantiate an one-layer feed-forward classifier\n",
        "        self.classifier = nn.Sequential(\n",
        "            nn.Linear(D_in, H),\n",
        "            nn.ReLU(),\n",
        "            #nn.Dropout(0.5),\n",
        "            nn.Linear(H, D_out)\n",
        "        )\n",
        "\n",
        "        # Freeze the BERT model\n",
        "        if freeze_bert:\n",
        "            for param in self.bert.parameters():\n",
        "                param.requires_grad = False\n",
        "\n",
        "    def forward(self, input_ids, attention_mask):\n",
        "        \"\"\"\n",
        "        Feed input to BERT and the classifier to compute logits.\n",
        "        @param    input_ids (torch.Tensor): an input tensor with shape (batch_size,\n",
        "                      max_length)\n",
        "        @param    attention_mask (torch.Tensor): a tensor that hold attention mask\n",
        "                      information with shape (batch_size, max_length)\n",
        "        @return   logits (torch.Tensor): an output tensor with shape (batch_size,\n",
        "                      num_labels)\n",
        "        \"\"\"\n",
        "        # Feed input to BERT\n",
        "        outputs = self.bert(input_ids=input_ids,\n",
        "                            attention_mask=attention_mask)\n",
        "\n",
        "        # Extract the last hidden state of the token `[CLS]` for classification task\n",
        "        last_hidden_state_cls = outputs[0][:, 0, :]\n",
        "\n",
        "        # Feed input to classifier to compute logits\n",
        "        logits = self.classifier(last_hidden_state_cls)\n",
        "\n",
        "        return logits"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 38 µs, sys: 0 ns, total: 38 µs\n",
            "Wall time: 40.1 µs\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LwNrCgPh-yR7"
      },
      "source": [
        "### 3.2. Optimizer & Learning Rate Scheduler"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V6iOXiN8-8gc"
      },
      "source": [
        "To fine-tune our Bert Classifier, we need to create an optimizer. The authors recommend following hyper-parameters:\n",
        "\n",
        "- Batch size: 16 or 32\n",
        "- Learning rate (Adam): 5e-5, 3e-5 or 2e-5\n",
        "- Number of epochs: 2, 3, 4\n",
        "\n",
        "Huggingface provided the [run_glue.py](https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L109) script, an examples of implementing the `transformers` library. In the script, the AdamW optimizer is used."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JX7su7Q_269U"
      },
      "source": [
        "from transformers import AdamW, get_linear_schedule_with_warmup\n",
        "\n",
        "def initialize_model(epochs=4):\n",
        "    \"\"\"Initialize the Bert Classifier, the optimizer and the learning rate scheduler.\n",
        "    \"\"\"\n",
        "    # Instantiate Bert Classifier\n",
        "    bert_classifier = BertClassifier(freeze_bert=False)\n",
        "\n",
        "    # Tell PyTorch to run the model on GPU\n",
        "    bert_classifier.to(device)\n",
        "\n",
        "    # Create the optimizer\n",
        "    optimizer = AdamW(bert_classifier.parameters(),\n",
        "                      lr=5e-5,    # Default learning rate\n",
        "                      eps=1e-8    # Default epsilon value\n",
        "                      )\n",
        "\n",
        "    # Total number of training steps\n",
        "    total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "    # Set up the learning rate scheduler\n",
        "    scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                                num_warmup_steps=0, # Default value\n",
        "                                                num_training_steps=total_steps)\n",
        "    return bert_classifier, optimizer, scheduler"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "41DRNjv4B0Ow"
      },
      "source": [
        "### 3.3. Training Loop"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VYU-GQRZG0y8"
      },
      "source": [
        "We will train our Bert Classifier for 4 epochs. In each epoch, we will train our model and evaluate its performance on the validation set. In more details, we will:\n",
        "\n",
        "Training:\n",
        "- Unpack our data from the dataloader and load the data onto the GPU\n",
        "- Zero out gradients calculated in the previous pass\n",
        "- Perform a forward pass to compute logits and loss\n",
        "- Perform a backward pass to compute gradients (`loss.backward()`)\n",
        "- Clip the norm of the gradients to 1.0 to prevent \"exploding gradients\"\n",
        "- Update the model's parameters (`optimizer.step()`)\n",
        "- Update the learning rate (`scheduler.step()`)\n",
        "\n",
        "Evaluation:\n",
        "- Unpack our data and load onto the GPU\n",
        "- Forward pass\n",
        "- Compute loss and accuracy rate over the validation set\n",
        "\n",
        "The script below is commented with the details of our training and evaluation loop."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xy4HkhyECibW"
      },
      "source": [
        "import random\n",
        "import time\n",
        "\n",
        "# Specify loss function\n",
        "loss_fn = nn.CrossEntropyLoss()\n",
        "\n",
        "def set_seed(seed_value=42):\n",
        "    \"\"\"Set seed for reproducibility.\n",
        "    \"\"\"\n",
        "    random.seed(seed_value)\n",
        "    np.random.seed(seed_value)\n",
        "    torch.manual_seed(seed_value)\n",
        "    torch.cuda.manual_seed_all(seed_value)\n",
        "\n",
        "def train(model, train_dataloader, val_dataloader=None, epochs=4, evaluation=False):\n",
        "    \"\"\"Train the BertClassifier model.\n",
        "    \"\"\"\n",
        "    # Start training loop\n",
        "    print(\"Start training...\\n\")\n",
        "    for epoch_i in range(epochs):\n",
        "        # =======================================\n",
        "        #               Training\n",
        "        # =======================================\n",
        "        # Print the header of the result table\n",
        "        print(f\"{'Epoch':^7} | {'Batch':^7} | {'Train Loss':^12} | {'Val Loss':^10} | {'Val Acc':^9} | {'Elapsed':^9}\")\n",
        "        print(\"-\"*70)\n",
        "\n",
        "        # Measure the elapsed time of each epoch\n",
        "        t0_epoch, t0_batch = time.time(), time.time()\n",
        "\n",
        "        # Reset tracking variables at the beginning of each epoch\n",
        "        total_loss, batch_loss, batch_counts = 0, 0, 0\n",
        "\n",
        "        # Put the model into the training mode\n",
        "        model.train()\n",
        "\n",
        "        # For each batch of training data...\n",
        "        for step, batch in enumerate(train_dataloader):\n",
        "            batch_counts +=1\n",
        "            # Load batch to GPU\n",
        "            b_input_ids, b_attn_mask, b_labels = tuple(t.to(device) for t in batch)\n",
        "\n",
        "            # Zero out any previously calculated gradients\n",
        "            model.zero_grad()\n",
        "\n",
        "            # Perform a forward pass. This will return logits.\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "\n",
        "            # Compute loss and accumulate the loss values\n",
        "            loss = loss_fn(logits, b_labels)\n",
        "            batch_loss += loss.item()\n",
        "            total_loss += loss.item()\n",
        "\n",
        "            # Perform a backward pass to calculate gradients\n",
        "            loss.backward()\n",
        "\n",
        "            # Clip the norm of the gradients to 1.0 to prevent \"exploding gradients\"\n",
        "            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "            # Update parameters and the learning rate\n",
        "            optimizer.step()\n",
        "            scheduler.step()\n",
        "\n",
        "            # Print the loss values and time elapsed for every 20 batches\n",
        "            if (step % 20 == 0 and step != 0) or (step == len(train_dataloader) - 1):\n",
        "                # Calculate time elapsed for 20 batches\n",
        "                time_elapsed = time.time() - t0_batch\n",
        "\n",
        "                # Print training results\n",
        "                print(f\"{epoch_i + 1:^7} | {step:^7} | {batch_loss / batch_counts:^12.6f} | {'-':^10} | {'-':^9} | {time_elapsed:^9.2f}\")\n",
        "\n",
        "                # Reset batch tracking variables\n",
        "                batch_loss, batch_counts = 0, 0\n",
        "                t0_batch = time.time()\n",
        "\n",
        "        # Calculate the average loss over the entire training data\n",
        "        avg_train_loss = total_loss / len(train_dataloader)\n",
        "\n",
        "        print(\"-\"*70)\n",
        "        # =======================================\n",
        "        #               Evaluation\n",
        "        # =======================================\n",
        "        if evaluation == True:\n",
        "            # After the completion of each training epoch, measure the model's performance\n",
        "            # on our validation set.\n",
        "            val_loss, val_accuracy = evaluate(model, val_dataloader)\n",
        "\n",
        "            # Print performance over the entire training data\n",
        "            time_elapsed = time.time() - t0_epoch\n",
        "\n",
        "            print(f\"{epoch_i + 1:^7} | {'-':^7} | {avg_train_loss:^12.6f} | {val_loss:^10.6f} | {val_accuracy:^9.2f} | {time_elapsed:^9.2f}\")\n",
        "            print(\"-\"*70)\n",
        "        print(\"\\n\")\n",
        "\n",
        "    print(\"Training complete!\")\n",
        "\n",
        "\n",
        "def evaluate(model, val_dataloader):\n",
        "    \"\"\"After the completion of each training epoch, measure the model's performance\n",
        "    on our validation set.\n",
        "    \"\"\"\n",
        "    # Put the model into the evaluation mode. The dropout layers are disabled during\n",
        "    # the test time.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables\n",
        "    val_accuracy = []\n",
        "    val_loss = []\n",
        "\n",
        "    # For each batch in our validation set...\n",
        "    for batch in val_dataloader:\n",
        "        # Load batch to GPU\n",
        "        b_input_ids, b_attn_mask, b_labels = tuple(t.to(device) for t in batch)\n",
        "\n",
        "        # Compute logits\n",
        "        with torch.no_grad():\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "\n",
        "        # Compute loss\n",
        "        loss = loss_fn(logits, b_labels)\n",
        "        val_loss.append(loss.item())\n",
        "\n",
        "        # Get the predictions\n",
        "        preds = torch.argmax(logits, dim=1).flatten()\n",
        "\n",
        "        # Calculate the accuracy rate\n",
        "        accuracy = (preds == b_labels).cpu().numpy().mean() * 100\n",
        "        val_accuracy.append(accuracy)\n",
        "\n",
        "    # Compute the average accuracy and loss over the validation set.\n",
        "    val_loss = np.mean(val_loss)\n",
        "    val_accuracy = np.mean(val_accuracy)\n",
        "\n",
        "    return val_loss, val_accuracy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BSfTy9LqiFD-"
      },
      "source": [
        "Now, let's start training our BertClassifier!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wfYw7dJ0U0v6",
        "outputId": "80f02200-4234-41de-dfd8-0a98e8fb6d32",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 574,
          "referenced_widgets": [
            "b7ce4c3233df4adfb0e914d5a445a019",
            "d581b0e01f654ac0b503fd58c5288348",
            "18d585f78629424fa2fc61feb2c2fd3a",
            "b0f389b162c94eb1b424e9c50092ec4c",
            "96ec7e0964e24e7da6c64d65ecc195e1",
            "280acaefa8d84013b50ed9238965b8e0",
            "e7f104c91014475cab28e59183a1ec1b",
            "308be4f1105141a2ac9885758d1dd1f2",
            "9b27ad8651b5415f93c401d5cda4a080",
            "9b298e31e3be4fcb96f83ef18a66b87a",
            "8f66a780bcc84b9fb14794cf3934408a",
            "eb8c20e05f204a97a06f5bd6284a0775",
            "47e45c9730574e15a7eeca7fddc99732",
            "b84700dfac5f4ad4b3100622a6148484",
            "af4d474d40af4605be6a399f920b134f",
            "28bb56ade0004bbd8f41916d4b27df40"
          ]
        }
      },
      "source": [
        "set_seed(42)    # Set seed for reproducibility\n",
        "bert_classifier, optimizer, scheduler = initialize_model(epochs=2)\n",
        "train(bert_classifier, train_dataloader, val_dataloader, epochs=2, evaluation=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b7ce4c3233df4adfb0e914d5a445a019",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=361, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9b27ad8651b5415f93c401d5cda4a080",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=440473133, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Start training...\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   1    |   20    |   0.630467   |     -      |     -     |   7.58   \n",
            "   1    |   40    |   0.497330   |     -      |     -     |   7.01   \n",
            "   1    |   60    |   0.502320   |     -      |     -     |   7.11   \n",
            "   1    |   80    |   0.491438   |     -      |     -     |   7.19   \n",
            "   1    |   95    |   0.486125   |     -      |     -     |   5.35   \n",
            "----------------------------------------------------------------------\n",
            "   1    |    -    |   0.524515   |  0.439601  |   78.81   |   35.54  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   2    |   20    |   0.287401   |     -      |     -     |   7.83   \n",
            "   2    |   40    |   0.260870   |     -      |     -     |   7.60   \n",
            "   2    |   60    |   0.287706   |     -      |     -     |   7.67   \n",
            "   2    |   80    |   0.283311   |     -      |     -     |   7.87   \n",
            "   2    |   95    |   0.280315   |     -      |     -     |   5.87   \n",
            "----------------------------------------------------------------------\n",
            "   2    |    -    |   0.279978   |  0.454067  |   80.40   |   38.31  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            "Training complete!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D5ostg9kPlra"
      },
      "source": [
        "### 3.4. Evaluation on Validation Set"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fIlSTDA7Z9DF"
      },
      "source": [
        "The prediction step is similar to the evaluation step that we did in the training loop, but simpler. We will perform a forward pass to compute logits and apply softmax to calculate probabilities."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V5_w4erqGzpe"
      },
      "source": [
        "import torch.nn.functional as F\n",
        "\n",
        "def bert_predict(model, test_dataloader):\n",
        "    \"\"\"Perform a forward pass on the trained BERT model to predict probabilities\n",
        "    on the test set.\n",
        "    \"\"\"\n",
        "    # Put the model into the evaluation mode. The dropout layers are disabled during\n",
        "    # the test time.\n",
        "    model.eval()\n",
        "\n",
        "    all_logits = []\n",
        "\n",
        "    # For each batch in our test set...\n",
        "    for batch in test_dataloader:\n",
        "        # Load batch to GPU\n",
        "        b_input_ids, b_attn_mask = tuple(t.to(device) for t in batch)[:2]\n",
        "\n",
        "        # Compute logits\n",
        "        with torch.no_grad():\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "        all_logits.append(logits)\n",
        "\n",
        "    # Concatenate logits from each batch\n",
        "    all_logits = torch.cat(all_logits, dim=0)\n",
        "\n",
        "    # Apply softmax to calculate probabilities\n",
        "    probs = F.softmax(all_logits, dim=1).cpu().numpy()\n",
        "\n",
        "    return probs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "outputId": "ca3adcfc-e833-462c-9368-b5d87905dca5",
        "id": "kcmj5s0eRMUh",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 329
        }
      },
      "source": [
        "# Compute predicted probabilities on the test set\n",
        "probs = bert_predict(bert_classifier, val_dataloader)\n",
        "\n",
        "# Evaluate the Bert classifier\n",
        "evaluate_roc(probs, y_val)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "AUC: 0.9048\n",
            "Accuracy: 80.59%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de7xVc/7H8ddHdEFCGWO6EIUukkRyKxFJiYnEoAwy7pf4jRkzGGNchnGd0BHDuNSQQS7JoCRUutFNpOhCJEVR6dTn98d3HWd3nLPPPpd91r68n4/Hfpy99lp77c9e55z92d/vd30/y9wdERGRsmwRdwAiIpLZlChERCQpJQoREUlKiUJERJJSohARkaSUKEREJCklCqkQM5ttZl3jjiNTmNkfzWxYTK/9iJndGMdrVzcz+42ZvVrJ5+pvMs2UKLKYmX1qZmvNbI2ZLYs+OLZN52u6ext3H5fO1yhiZnXM7GYzWxS9z4/N7Cozs5p4/VLi6WpmSxIfc/eb3P2cNL2emdklZjbLzL43syVm9rSZ7ZOO16ssM7vezB6vyj7c/Ql3PzqF1/pZcqzJv8l8pUSR/Xq7+7ZAe2A/4A8xx1NhZrZlGaueBo4EegL1gTOAQcDdaYjBzCzT/h/uBi4FLgF2BPYEngOOq+4XSvI7SLs4X1tS5O66ZekN+BQ4KmH578BLCcsHAe8Aq4D3ga4J63YE/gV8DqwEnktY1wuYET3vHaBdydcEfgWsBXZMWLcf8DWwVbT8W2ButP8xwK4J2zpwIfAxsLCU93YksA5oWuLxTsBGoEW0PA64GZgMfAc8XyKmZMdgHPA34O3ovbQAzopiXg0sAM6Ltt0m2mYTsCa6/Qq4Hng82ma36H0NABZFx+KahNerBzwaHY+5wP8BS8r43baM3ueBSX7/jwBDgJeieCcBeySsvxtYHB2XqcBhCeuuB0YCj0frzwEOBN6NjtUXwD+B2gnPaQP8D/gG+BL4I9AD+BHYEB2T96NtGwAPRftZCtwI1IrWDYyO+Z3AimjdQGBCtN6idV9Fsc0E2hK+JGyIXm8N8ELJ/wOgVhTXJ9ExmUqJvyHdKvFZE3cAulXhl7f5P0iT6B/q7mi5cfRP2JPQcuweLe8UrX8J+A+wA7AV0CV6fL/oH7RT9E83IHqdOqW85hvAuQnx3AY8EN3vA8wHWgFbAn8C3knY1qMPnR2BeqW8t1uAN8t4359R/AE+Lvogakv4MH+G4g/u8o7BOMIHepsoxq0I39b3iD6sugA/AB2i7btS4oOd0hPFg4SksC+wHmiV+J6iY94E+KDk/hL2+zvgs3J+/49E7+fAKP4ngBEJ608HGkbrBgPLgLoJcW8AToiOTT1gf0Ji3TJ6L3OBy6Lt6xM+9AcDdaPlTiWPQcJrPwsMjX4nvyAk8qLf2UCgELg4eq16bJ4ojiF8wG8f/R5aAbskvOcbk/wfXEX4P9greu6+QMO4/1ez/RZ7ALpV4ZcX/kHWEL45OfA6sH207vfAYyW2H0P44N+F8M14h1L2eT/w1xKPzaM4kST+U54DvBHdN8K318Oj5dHA2Qn72ILwobtrtOxAtyTvbVjih16JdROJvqkTPuxvSVjXmvCNs1ayY5Dw3BvKOcbPAZdG97uSWqJokrB+MtA/ur8AOCZh3Tkl95ew7hpgYjmxPQIMS1juCXyYZPuVwL4JcY8vZ/+XAc9G908Fppex3U/HIFremZAg6yU8diowNro/EFhUYh8DKU4U3YCPCElri1Lec7JEMQ/ok47/t3y+ZVqfrFTcCe5en/AhtjfQKHp8V+BkM1tVdAMOJSSJpsA37r6ylP3tCgwu8bymhG6Wkp4BOpvZLsDhhOTzVsJ+7k7YxzeEZNI44fmLk7yvr6NYS7NLtL60/XxGaBk0IvkxKDUGMzvWzCaa2TfR9j0pPqapWpZw/weg6ASDX5V4vWTvfwVlv/9UXgszu9LM5prZt9F7acDm76Xke9/TzF6MToz4DrgpYfumhO6cVOxK+B18kXDchxJaFqW+diJ3f4PQ7TUE+MrMCsxsuxRfuyJxSoqUKHKEu79J+LZ1e/TQYsK36e0Tbtu4+y3Ruh3NbPtSdrUY+FuJ523t7sNLec2VwKvAKcBphBaAJ+znvBL7qefu7yTuIslbeg3oZGZNEx80s06ED4M3Eh5O3KYZoUvl63KOwc9iMLM6hOR3O7Czu28PvExIcOXFm4ovCF1OpcVd0utAEzPrWJkXMrPDCGMg/Qgtx+2Bbyl+L/Dz93M/8CHQ0t23I/T1F22/GNi9jJcruZ/FhBZFo4Tjvp27t0nynM136H6Pu+9PaCHuSehSKvd50WvvUc42UkFKFLnlLqC7me1LGKTsbWbHmFktM6sbnd7ZxN2/IHQN3WdmO5jZVmZ2eLSPB4HfmVmn6EygbczsODOrX8ZrPgmcCZwU3S/yAPAHM2sDYGYNzOzkVN+Iu79G+LB8xszaRO/hoOh93e/uHydsfrqZtTazrYEbgJHuvjHZMSjjZWsDdYDlQKGZHQsknrL5JdDQzBqk+j5KeIpwTHYws8bARWVtGL2/+4DhUcy1o/j7m9nVKbxWfcI4wHJgSzO7FijvW3l9wuDxGjPbGzg/Yd2LwC5mdll02nL9KGlDOC67FZ01Fv19vQr8w8y2M7MtzGwPM+uSQtyY2QHR399WwPeEkxo2JbxWWQkLQpflX82sZfT3287MGqbyulI2JYoc4u7LgX8D17r7YsKA8h8JHxaLCd/Kin7nZxC+eX9IGLy+LNrHFOBcQtN/JWFAemCSlx1FOENnmbu/nxDLs8CtwIioG2MWcGwF31JfYCzwCmEs5nHCmTQXl9juMUJrahlhoPWSKIbyjsFm3H119NynCO/9tOj9Fa3/EBgOLIi6VErrjkvmBmAJsJDQYhpJ+OZdlkso7oJZRehSORF4IYXXGkM4bh8RuuPWkbyrC+BKwnteTfjC8J+iFdGx6Q70Jhznj4EjotVPRz9XmNm06P6ZhMQ7h3AsR5JaVxqEhPZg9LzPCN1wt0XrHgJaR8f/uVKeewfh9/cqIek9RBgslyqw4p4CkexjZuMIA6mxzI6uCjM7nzDQndI3bZG4qEUhUkPMbBczOyTqitmLcKrps3HHJVKetCUKM3vYzL4ys1llrDczu8fM5pvZB2bWIV2xiGSI2oSzf1YTBuOfJ4xDiGS0tHU9RYOja4B/u3vbUtb3JPQ19yRM7rrb3TuV3E5EROKVthaFu48nnDtflj6EJOLuPhHYPjofX0REMkicxbgas/lZGEuix74ouaGZDSLUeWGbbbbZf++9966RAEWk4ubNg7VroZ7ONcoIO6//jG0LV/G+F37t7jtVZh9ZUbXR3QuAAoCOHTv6lClTYo5IpGoKCuDJJ8vfLhvVqgWHHgrjxsUdSR4rGlIwg/vvh6++wq6//rPK7i7Os56WsvnM1CbRYyI5q6AAunaF886DN9+MO5r0aN8eTjst7ijy2NKl0KdP8TeR88+H666r0i7jbFGMAi4ysxGEwexvoxmdIlkn1RZCUXLo0iV8mA4alN64JI+4w7BhcOWVsGEDHFd9ly1JW6Iws+GEQnWNLFwV7DpCoTDc/QFCDZ2ehJm/PxCuAyCSlZ58EmbMCN+mk1GCkLT45BM491wYOxaOOAIefBD2qL6SV2lLFO5+ajnriy5cIxKr6hgvKEoS6peXWMycCVOnhj/mc84JYxPVKCsGs0WqorxEkNgdVFnql5caN2sWTJsGZ54JJ5wACxZAw/TUP1SikJxVlCDKSwTqDpKs8uOPcNNN4bbzztCvH9Stm7YkAUoUkmUq0k2kgWPJOZMmwdlnw+zZcPrpcOedIUmkmRKFZJVUB41BCUJyzNKlcNhhoRXx4ovVelZTeZQoJOto0FjyykcfwZ57QuPG8J//wJFHwnapXhm2eqjMuIhIJlq1KjSH994bxo8Pj514Yo0nCVCLQjJcyTGJVLudRLLaqFFhRvWyZXDVVXDAAbGGoxaFZLSiMYkiOg1Vct4554QSHA0bhsHrW2+NvcKiWhSSVlWdzKaJbJIXEov4dewIu+4Kv/891K4db1wRJQpJi1TnMJRHLQjJeYsXw+9+B/37wxlnhPsZRolCqlVpCUKnqIqUYtMmGDo0tBw2bgwD1RlKiUKqhRKESAV8/HEYixg/Ho46KvwDNW8ed1RlUqKQalE06KwEIZKCOXPggw/g4Ydh4MBqL+JX3ZQopEqKWhIadBYpx/vvh3+UAQPCWU0LFsAOO8QdVUqUKKRSyupqEpES1q+HG2+EW26BXXaBU04J9ZmyJEmAEoWkoLRTXDUWIZKCd98NRfzmzg3lwO+4o0aK+FU3JQopV2mF+JQgRMqxdGn4R/nlL+Hll+HYY+OOqNKUKKRMGn8QqYS5c6FVq1DE76mnQhG/+vXjjqpKlCjkJyW7mDT+IFIBK1fC4MHwr3+F014POyxceS4HKFHkobLKapScRa3uJZEUPfssXHABLF8Of/hD7EX8qpsSRR4q6+I/SgwilfDb34ZWRPv28NJL0KFD3BFVOyWKPKUxB5EqSCzid9BB0LIlXHklbLVVvHGliRKFiEhFfPYZnHdeaH6feWZeNMF1PYo8UlAAXbtufn0HEUnRpk0wZAi0bQsTJsCGDXFHVGPUosgDmkUtUkXz5oUifhMmwNFHh6qvu+0Wd1Q1RokiD6hgn0gVzZsHs2fDI4+E7qYML+JX3ZQocpgmzIlUwfTp4Z/nrLPg+ONDEb/tt487qlgoUWShVC8vqq4mkUpYtw5uuAH+/vcwu/rUU0N9pjxNEqDB7KxU1EooT5cuoSt13Dh1N4mk5O23Q/P75ptDF9OMGVlZxK+6qUWRpdSVJFLNli6FI44IrYgxY8KgtQBqUYhIvpszJ/xs3BieeQZmzlSSKEGJIgsUzX8oumkehEg1+OabcBnSNm1CET+A3r1h221jDSsTKVFkgZJjEu3ba3BapEqeeQZat4YnnoBrroEDD4w7ooymMYoMptNbRdJg4EB49NFQvO+VV35eHVN+RokigyUmCbUgRKogsYjfwQeHCwsNHgxb6iMwFWk9SmbWA7gbqAUMc/dbSqxvBjwKbB9tc7W7v5zOmLKNWhIiVbRwYTg//PTTYcAAnSteCWkbozCzWsAQ4FigNXCqmbUusdmfgKfcfT+gP3BfuuLJJireJ1INNm6Ee+4JRfwmTixuVUiFpXMw+0BgvrsvcPcfgRFAnxLbOLBddL8B8Hka48ka6nISqaK5c8OlSC+9NMw8nT07jE1IpaSz66kxsDhheQnQqcQ21wOvmtnFwDbAUaXtyMwGAYMAmjVrVu2BZiJ1OYlUwfz5oZDfY4/Bb36Td0X8qlvcp8eeCjzi7k2AnsBjZvazmNy9wN07unvHnXbaqcaDFJEsMHUqPPxwuN+7dxibOP10JYlqkM5EsRRomrDcJHos0dnAUwDu/i5QF2iUxpgyXkFBcTE/EUnB2rVw9dXQqRP89a+hqB/Adtslf56kLJ2J4j2gpZk1N7PahMHqUSW2WQQcCWBmrQiJYnkaY8pYRQPY550XljU2IZKC8eNh333h1lvDGMT06SrilwZpG6Nw90IzuwgYQzj19WF3n21mNwBT3H0UMBh40MwuJwxsD3TPz1MTdHEhkQpauhSOPBKaNoXXXgv3JS3SOo8imhPxconHrk24Pwc4JJ0xZBMNYIukYOZM2GefUMTv2WdDxddttok7qpwW92B23lKhP5EK+vprOOMMaNeuuIhfr15KEjVAiSImKvQnkiJ3eOqpUMRvxAi47rowcC01RoVOapgK/YlU0IABYT5Ex47w+uuh20lqlBJFDdOsa5EUJBbx69IldDdddpmK+MVER72GqCUhkqIFC+Dcc8NkubPOgrPPjjuivKdEkSZFiaFI0SS6otNfRaSEjRvh3nvDhYRq1YIzz4w7IokoUaRJYusBND9CJKk5c+C3v4VJk+C44+CBB6BJk7ijkogSRRqpi0kkRQsXwiefhG9Y/furPlOGUaIQkXi8915odp97bmhFLFgA9evHHZWUQomikkqOQZSU2O0kIgl++AGuvRbuvBN23TVMoqtbV0kig2nCXSWVnDBXkk5/FSnFuHHhVNd//CO0JFTELyuoRVEFGoMQqYAlS6B799CKeOONUKNJsoJaFCKSXu+/H342aQLPPw8ffKAkkWWUKEQkPZYvD/2v7dsXTyTq2RO23jreuKTC1PUkItXLPRTvu+QS+PZb+MtfoHPnuKOSKlCiKEdZZzfprCaRMpxxBjzxRKjw+tBD0KZN3BFJFaXc9WRmedleLOvsJp3VJJJg06biQn5HHAF33AFvv60kkSPKbVGY2cHAMGBboJmZ7Quc5+4XpDu4TKGzm0SSmD8/nOp6xhmhDIeK+OWcVFoUdwLHACsA3P194PB0BiUiWaCwEG6/PVwfYvp0qF077ogkTVIao3D3xbZ57ZWN6QlHRLLCrFmhBPiUKdCnD9x3H/zqV3FHJWmSSqJYHHU/uZltBVwKzE1vWCKS0RYtgs8+C2c39eunIn45LpWup98BFwKNgaVAeyDnxycKCqBr1+RlOkTyyqRJ4R8DwnyIBQvglFOUJPJAKi2Kvdz9N4kPmNkhwNvpCSleRafD6kJDIpHvv4c//xnuugt23z1cw7pOHdh227gjkxqSSqK4F+iQwmNZrawEoQsNSV57441wRtOCBXD++XDLLSFJSF4pM1GYWWfgYGAnM7siYdV2QK10B1ZTlCBEyrBkCRxzDDRvHv5BDtfJjvkqWYuiNmHuxJZAYqH474CT0hlUTSqaUKcEIRKZPh322y8U8XvhhfDPUa9e3FFJjMpMFO7+JvCmmT3i7p/VYEw1pqAgfFHq0kUT6kT48stQn+mpp8I/RJcu0KNH3FFJBkhljOIHM7sNaAP8dIURd++WtqhqSFENJw1WS15zD7WZLr0U1qyBG2+Egw+OOyrJIKmcHvsE8CHQHPgL8CnwXhpjqlFduqi7SfLcaaeF8ht77RX6Ya+5BrbaKu6oJIOk0qJo6O4PmdmlCd1ROZMoRPLSpk1h/oMZHH10KAN+4YVQK2fOU5FqlEqLYkP08wszO87M9gN2TGNMIpJOH30UKrw+/HBYPuusMDahJCFlSKVFcaOZNQAGE+ZPbAdcltaoRKT6FRaG8t/XXQd16+pMJklZuYnC3V+M7n4LHAE/zcwWkWzxwQehBPjUqXDiiTBkCOyyS9xRSZZINuGuFtCPUOPpFXefZWa9gD8C9YD9aiZEEamyJUtg8WJ4+mno21f1maRCko1RPAScAzQE7jGzx4Hbgb+7e0pJwsx6mNk8M5tvZleXsU0/M5tjZrPNrJSLjopIpbzzDjzwQLhfVMTvpJOUJKTCknU9dQTaufsmM6sLLAP2cPcVqew4apEMAboDS4D3zGyUu89J2KYl8AfgEHdfaWa/qOwbEZHImjXhFNd774U99giD1XXqwDbbxB2ZZKlkLYof3X0TgLuvAxakmiQiBwLz3X2Bu/8IjAD6lNjmXGCIu6+MXuerCuxfREp69VVo2zYkiQsvhGnTVMRPqixZi2JvM/sgum/AHtGyAe7u7crZd2NgccLyEqBTiW32BDCztwmFBq9391dK7sjMBgGDAJo1a1bOy4rkqcWL4bjjQiti/Hg49NC4I5IckSxRtKqh128JdAWaAOPNbB93X5W4kbsXAAUAHTt29BqISyR7TJ0K++8PTZvCyy/DYYeF019FqkmZXU/u/lmyWwr7Xgo0TVhuEj2WaAkwyt03uPtC4CNC4hCR8ixbBiefDB07FtfJ795dSUKqXSozsyvrPaClmTU3s9pAf2BUiW2eI7QmMLNGhK6oBWmMSST7ucOjj0Lr1qEM+E03qYifpFUqM7Mrxd0LzewiYAxh/OFhd59tZjcAU9x9VLTuaDObA2wErqrggLlI/unfP5QCP+QQGDYM9t477ogkx6WUKMysHtDM3edVZOfu/jLwconHrk2478AV0U1EypJYxK9nzzAOccEFsEU6OwVEgnL/ysysNzADeCVabm9mJbuQRCRdPvwwXIb0oYfC8oABcNFFShJSY1L5S7ueMCdiFYC7zyBcm0JE0mnDhjD+sO++MGcObLtt3BFJnkql62mDu39rm0/71ymqIuk0Y0aYUT1jRii7ce+98Mtfxh2V5KlUEsVsMzsNqBWV3LgEeCe9YYnkuWXLwu2ZZ+DXv447GslzqXQ9XUy4XvZ64ElCuXFdj0Kkuk2YAPfdF+736AGffKIkIRkhlUSxt7tf4+4HRLc/RbWfRKQ6rF4dBqcPOwzuugvWrw+Pb711vHGJRFJJFP8ws7lm9lcza5v2iETyyZgxoYjffffBpZeqiJ9kpHIThbsfQbiy3XJgqJnNNLM/pT0ykVy3eDH06hVaDhMmhNaEzmySDJTSidjuvszd7wF+R5hTcW05TxGR0rjD5MnhftOmMHo0TJ+uEhyS0VKZcNfKzK43s5nAvYQznpqkPTKRXPPFF+EypJ06FRfxO+ooFfGTjJdKi+JhwmS7Y9y9q7vfn+0XGCoogK5dwynqImnnDv/6VyjiN3o03HprqNMkkiXKnUfh7p1rIpCa9OSTIUm0bw+nnRZ3NJLz+vWDkSPDWU3DhsGee8YdkUiFlJkozOwpd+8XdTklzsRO9Qp3Ga19exg3Lu4oJGdt3BgK+G2xBfTuDd26wXnnqT6TZKVkLYpLo5+9aiIQkZwxdy6cfXYowXHuuXDmmXFHJFIlya5w90V094JSrm53Qc2EJ5JFNmyAG28MzdV586BBg7gjEqkWqbSDu5fy2LHVHYhIVps+PVyS9M9/hhNPDK2Kfv3ijkqkWiQbozif0HLY3cw+SFhVH3g73YGJZJUvv4Svv4bnnoM+feKORqRaJRujeBIYDdwMXJ3w+Gp3/yatUYlkg/HjYeZMuPDCUMRv/nyoVy/uqESqXbKuJ3f3T4ELgdUJN8xsx/SHJpKhvvsuXIa0Sxe4557iIn5KEpKjkiWKJ6OfU4Ep0c+pCctZRxPtpMpefhnatIGhQ+GKK1TET/JCmV1P7t4r+pkzlz3VRDupksWLw/jDXnuFCXSdOsUdkUiNKHdmtpkdAsxw9+/N7HSgA3CXuy9Ke3RpoIl2UiHuMGkSHHRQKOL36quh/Ebt2nFHJlJjUjk99n7gBzPbFxgMfAI8ltaoRDLB55/DCSdA587FRfyOOEJJQvJOKomi0N0d6AP8092HEE6RzRoam5AKcQ81mVq3Di2I229XET/Ja+V2PQGrzewPwBnAYWa2BbBVesOqHgUFYVyi6Mtgly4am5AUnHQS/Pe/4Q9m2DBo0SLuiERilUqiOAU4Dfituy8zs2bAbekNq3oUDV4XJYhBg+KOSDJWYhG/E06Ao48OdZpUxE8kpTLjy8zsCeAAM+sFTHb3f6c/tOqhwWsp16xZcM45oZDfuefCGWfEHZFIRknlCnf9gMnAyUA/YJKZnZTuwETS7scf4S9/gQ4d4JNPYIcd4o5IJCOl0vV0DXBA0VXtzGwn4DVgZDoDq4qisYmiORMiPzN1KgwcGFoTp50Gd90FO+0Ud1QiGSmVRLFFiUufriC1s6Vio4l1Uq4VK2DVKnjhBeilS66IJJNKonjFzMYAw6PlU4CX0xdS9dDYhPzM2LGhiN8ll4TB6o8/hrp1445KJOOV2zJw96uAoUC76Fbg7r9Pd2CVofkSUqpvvw2XIe3WDe6/v7iIn5KESEqSXY+iJXA7sAcwE7jS3ZfWVGCVoS4n+ZkXXoDf/Q6WLYMrrwyD1yriJ1IhybqeHgb+DYwHegP3Ar+uiaCqQl1O8pPFi6FvX9h773BBoQMOiDsikayULFHUd/cHo/vzzGxaTQQkUiXu8O67cPDBxUX8Dj5Y9ZlEqiDZGEVdM9vPzDqYWQegXonlcplZDzObZ2bzzezqJNv1NTM3s44VfQMiP1myBI4/PtRlKqrb0rWrkoRIFSVrUXwB3JGwvCxh2YFuyXZsZrWAIUB3YAnwnpmNcvc5JbarD1wKTKpY6CKRTZvgwQfhqqugsBDuuAMOPTTuqERyRrILFx1RxX0fCMx39wUAZjaCUIF2Tont/grcClxV2RfSBLs817dvGIPo1i0kjN13jzsikZySzolzjYHFCctLosd+EnVhNXX3l5LtyMwGmdkUM5uyfPnyn63X2U55qLAwtCQgJIoHH4TXXlOSEEmDVCbcpUVUrvwOYGB527p7AVAA0LFjRy9tG53tlEc++CAU8DvnnDA/4vTT445IJKels0WxFGiasNwkeqxIfaAtMM7MPgUOAkZpQFvKtH49XHcd7L8/fPaZajOJ1JBUqseamZ1uZtdGy83M7MAU9v0e0NLMmptZbaA/MKpopbt/6+6N3H03d98NmAgc7+5TKvVOJLe9916o8nrDDXDqqTB3Lvw646f1iOSEVFoU9wGdgVOj5dWEs5mScvdC4CJgDDAXeMrdZ5vZDWZ2fCXjlXy1ciWsWQMvvwz//jc0bBh3RCJ5I5Uxik7u3sHMpgO4+8qohVAud3+ZEgUE3f3aMrbtmso+JY+88UYo4nfppaGI30cfqfyGSAxSaVFsiOZEOPx0PYpNaY1K8tuqVeFKc0ceCUOHFhfxU5IQiUUqieIe4FngF2b2N2ACcFNao5L89fzz0Lo1PPww/N//hQsMKUGIxCqVa2Y/YWZTgSMBA05w97lpj0zyz6JFcPLJ0KoVjBoFHXUCnEgmKDdRmFkz4AfghcTH3H1ROgOTPOEOEybAYYdBs2Zh0txBB6k+k0gGSWUw+yXC+IQBdYHmwDygTRrjknywaFG4VsTo0WG2ZJcucPjhcUclIiWkcoW7fdy9XfSzJaGG07vpD618uqJdltq0Ce67D9q0gfHj4Z57VMRPJINVuISHu08zs07pCKaiVOMpS/3612HQunv3kO132y3uiEQkiVTGKK5IWNwC6AB8nraIKkg1nrJEYSFssUW4nXIK9OkDAweCWdyRiUg5Ujk9tn7CrQ5hzKJPOoOSHPP++9CpU2g9QCjBcdZZShIiWSJpiyKaaFff3a+soXgkl6xbBzfeCLfeCjvuCL/8ZdwRiUgllJkozGxLdy80s0NqMiDJEZMnw4AB8OGH4dzY5tcAABPuSURBVOcdd4RkISJZJ1mLYjJhPGKGmY0Cnga+L1rp7v9Nc2ySzb77DtauhVdegWOOiTsaEamCVM56qgusIFwju2g+hQNKFLK5V1+F2bPh8svhqKNg3jyV3xDJAckSxS+iM55mUZwgipR6lTnJUytXwhVXwCOPhLkRF1wQEoSShEhOSHbWUy1g2+hWP+F+0U0E/vvfUMTvscfgD3+AKVOUIERyTLIWxRfufkONRVIBBQWbT7aTmCxaBP37Q9u24YJC++0Xd0QikgbJWhQZe5K7ZmTHyB3efDPcb9YsXFxo0iQlCZEclqxFcWSNRVEJmpEdg88+g/POgzFjiov4qUaTSM4rs0Xh7t/UZCCSwTZtgn/+MwxUT5gA994byoKLSF6ocFFAyUMnnAAvvBDmQwwdCrvuGndEIlKDlCikdBs2QK1aoYjfqafCSSfBGWeoPpNIHkqlKKDkm2nT4MAD4YEHwvKpp8KZZypJiOSprEsU8+bpQkVps3ZtmAtx4IGwbBk0bRp3RCKSAbKu62nt2nCijU6LrWYTJ4bifR99BL/9Ldx+O+ywQ9xRiUgGyLpEUa+eTotNi++/D+MS//tfqNMkIhLJukQh1eiVV0IRv8GD4cgjQ0nw2rXjjkpEMkzWjVFINVixInQzHXssPPoo/PhjeFxJQkRKoUSRT9xh5MhQxO/JJ+FPf4L33lOCEJGk1PWUTxYtCmcBtGsXrh2x775xRyQiWUAtilznHgr3QZhRPW5cOMNJSUJEUqREkcsWLoSjjw4D1UUVXw8+GLZUQ1JEUqdEkYs2boS77w7XiZg0Ce6/X0X8RKTS9NUyF/XpAy+9BD17hjIcmmEtIlWgRJErEov4nXFGqM902mmqzyQiVZbWricz62Fm88xsvpldXcr6K8xsjpl9YGavm5nqV1fGlCnQsWPoYgI45RT4zW+UJESkWqQtUZhZLWAIcCzQGjjVzFqX2Gw60NHd2wEjgb+nK56ctHYt/P730KkTLF+u60SISFqks0VxIDDf3Re4+4/ACKBP4gbuPtbdf4gWJwJN0hhPbnn33XCK69//Hor4zZkDvXrFHZWI5KB0jlE0BhYnLC8BOiXZ/mxgdGkrzGwQMAigTp121RVfdlu7Nlyi9LXXwumvIiJpkhGD2WZ2OtAR6FLaencvAAoA6tfv6DUYWmZ5+eVQxO+qq6BbN5g7F7baKu6oRCTHpbPraSmQeF5mk+ixzZjZUcA1wPHuvj6N8WSvr7+G00+H446DJ54oLuKnJCEiNSCdieI9oKWZNTez2kB/YFTiBma2HzCUkCS+SmMs2ckdRoyAVq3gqafguutg8mQV8RORGpW2rid3LzSzi4AxQC3gYXefbWY3AFPcfRRwG7At8LSFUzkXufvx6Yop6yxaFMqB77svPPQQ7LNP3BGJSB4y9+zq8q9fv6OvXj0l7jDSxx1ef734KnMTJ8IBB4TJdCIilWRmU929Y2Weq1pPmeSTT8IZTN27FxfxO+ggJQkRiZUSRSbYuBHuuCN0LU2dCkOHqoifiGSMjDg9Nu/17g2jR4cJc/ffD00071BEMocSRVx+/DFcF2KLLWDgwFDIr39/1WcSkYyjrqc4TJ4M++8P990Xlvv1C9VelSREJAMpUdSkH36AwYOhc2dYuRL22CPuiEREyqWup5oyYUKYE7FgAZx3Htx6KzRoEHdUIiLlUqKoKUUXFho7Frp2jTsaEZGUKVGk0wsvhMJ9//d/cMQRoRT4ljrkIpJdNEaRDsuXh8uQHn88DB9eXMRPSUJEspASRXVyhyefDEX8Ro6EG26ASZNUxE9Espq+4lanRYvgrLNgv/1CEb82beKOSESkytSiqKpNm2DMmHB/113hrbfg7beVJEQkZyhRVMXHH4crzfXoAePHh8cOPFBF/EQkpyhRVEZhIdx2G7RrBzNmhG4mFfETkRylMYrK6NUrdDf16RPKcPzqV3FHJJKRNmzYwJIlS1i3bl3coeSNunXr0qRJE7aqxksl68JFqVq/PlyjeostwhlNmzbBySerPpNIEgsXLqR+/fo0bNgQ0/9K2rk7K1asYPXq1TRv3nyzdbpwUbpNnAgdOsCQIWH5pJNCIT/94YsktW7dOiWJGmRmNGzYsNpbcEoUyXz/PVx+ORx8MKxeDS1bxh2RSNZRkqhZ6TjeGqMoy1tvhSJ+CxfCBRfAzTfDdtvFHZWISI1Ti6IshYVhTOLNN0OXk5KESNZ67rnnMDM+/PDDnx4bN24cvXr12my7gQMHMnLkSCAMxF999dW0bNmSDh060LlzZ0aPHl3lWG6++WZatGjBXnvtxZiiOVglvPHGG3To0IG2bdsyYMAACgsLgTAGcckll9CiRQvatWvHtGnTqhxPKpQoEj33XGg5QCjiN3s2HH54vDGJSJUNHz6cQw89lOHDh6f8nD//+c988cUXzJo1i2nTpvHcc8+xevXqKsUxZ84cRowYwezZs3nllVe44IIL2Lhx42bbbNq0iQEDBjBixAhmzZrFrrvuyqOPPgrA6NGj+fjjj/n4448pKCjg/PPPr1I8qVLXE8CXX8LFF8PTT4dB68GDQ30mFfETqTaXXRamHVWn9u3hrruSb7NmzRomTJjA2LFj6d27N3/5y1/K3e8PP/zAgw8+yMKFC6lTpw4AO++8M/369atSvM8//zz9+/enTp06NG/enBYtWjB58mQ6d+780zYrVqygdu3a7LnnngB0796dm2++mbPPPpvnn3+eM888EzPjoIMOYtWqVXzxxRfssssuVYqrPPndonCHxx6D1q3h+efhb38LZzipiJ9Iznj++efp0aMHe+65Jw0bNmTq1KnlPmf+/Pk0a9aM7VLocr788stp3779z2633HLLz7ZdunQpTZs2/Wm5SZMmLF26dLNtGjVqRGFhIVOmhGkAI0eOZPHixSk/Px3y+yvzokVwzjnQsWOYXb333nFHJJKzyvvmny7Dhw/n0ksvBaB///4MHz6c/fffv8yzgyp61tCdd95Z5RhLvv6IESO4/PLLWb9+PUcffTS1Yi4LlH+JoqiI37HHhiJ+b78dqr2qPpNIzvnmm2944403mDlzJmbGxo0bMTNuu+02GjZsyMqVK3+2faNGjWjRogWLFi3iu+++K7dVcfnllzN27NifPd6/f3+uvvrqzR5r3LjxT60DgCVLltC4ceOfPbdz58689dZbALz66qt89NFHFXp+tXP3rLptu+3+Xmnz5rkfdpg7uI8bV/n9iEhK5syZE+vrDx061AcNGrTZY4cffri/+eabvm7dOt9tt91+ivHTTz/1Zs2a+apVq9zd/aqrrvKBAwf6+vXr3d39q6++8qeeeqpK8cyaNcvbtWvn69at8wULFnjz5s29sLDwZ9t9+eWX7u6+bt0679atm7/++uvu7v7iiy96jx49fNOmTf7uu+/6AQccUOrrlHbcgSleyc/d/BijKCyEW28NRfxmzoR//UtnM4nkgeHDh3PiiSdu9ljfvn0ZPnw4derU4fHHH+ess86iffv2nHTSSQwbNowGDRoAcOONN7LTTjvRunVr2rZtS69evVIas0imTZs29OvXj9atW9OjRw+GDBnyU7dSz549+fzzzwG47bbbaNWqFe3ataN3795069btp2123313WrRowbnnnst9991XpXhSlR+1no45Bl59FX796zAn4pe/TE9wIrKZuXPn0qpVq7jDyDulHfeq1HrK3TGKdevChLlatWDQoHDr2zfuqEREsk5udj29/XY4wbqoiF/fvkoSIiKVlFuJYs0auOSScBGhdetATV6R2GVb93a2S8fxzp1E8eab0LYt/POfcNFFMGsWdO8ed1Qiea1u3bqsWLFCyaKGeHQ9irp161brfnNrjGLrrUPV10MOiTsSESHMHF6yZAnLly+PO5S8UXSFu+qU3Wc9/fe/8OGH8Mc/huWNGzVxTkSkFBl7hTsz62Fm88xsvpldXcr6Omb2n2j9JDPbLaUdL1sWrjLXty88+yz8+GN4XElCRKTapS1RmFktYAhwLNAaONXMWpfY7Gxgpbu3AO4Ebi1vvw02rAiD1C++GEqCv/OOiviJiKRROlsUBwLz3X2Bu/8IjAD6lNimD/BodH8kcKSVU5Fr5/WfhUHr99+Hq68OcyVERCRt0jmY3RhYnLC8BOhU1jbuXmhm3wINga8TNzKzQcCgaHG9TZgwS5VeAWhEiWOVx3QsiulYFNOxKLZXZZ+YFWc9uXsBUABgZlMqOyCTa3QsiulYFNOxKKZjUczMKlj7qFg6u56WAk0TlptEj5W6jZltCTQAVqQxJhERqaB0Jor3gJZm1tzMagP9gVElthkFDIjunwS84dl2vq6ISI5LW9dTNOZwETAGqAU87O6zzewGQl30UcBDwGNmNh/4hpBMylOQrpizkI5FMR2LYjoWxXQsilX6WGTdhDsREalZuVPrSURE0kKJQkREksrYRJG28h9ZKIVjcYWZzTGzD8zsdTPbNY44a0J5xyJhu75m5maWs6dGpnIszKxf9Lcx28yerOkYa0oK/yPNzGysmU2P/k96xhFnupnZw2b2lZnNKmO9mdk90XH6wMw6pLTjyl5sO503wuD3J8DuQG3gfaB1iW0uAB6I7vcH/hN33DEeiyOAraP75+fzsYi2qw+MByYCHeOOO8a/i5bAdGCHaPkXcccd47EoAM6P7rcGPo077jQdi8OBDsCsMtb3BEYDBhwETEplv5naokhL+Y8sVe6xcPex7v5DtDiRMGclF6XydwHwV0LdsHU1GVwNS+VYnAsMcfeVAO7+VQ3HWFNSORYObBfdbwB8XoPx1Rh3H084g7QsfYB/ezAR2N7Mdilvv5maKEor/9G4rG3cvRAoKv+Ra1I5FonOJnxjyEXlHouoKd3U3V+qycBikMrfxZ7Anmb2tplNNLMeNRZdzUrlWFwPnG5mS4CXgYtrJrSMU9HPEyBLSnhIaszsdKAj0CXuWOJgZlsAdwADYw4lU2xJ6H7qSmhljjezfdx9VaxRxeNU4BF3/4eZdSbM32rr7pviDiwbZGqLQuU/iqVyLDCzo4BrgOPdfX0NxVbTyjsW9YG2wDgz+5TQBzsqRwe0U/m7WAKMcvcN7r4Q+IiQOHJNKsfibOApAHd/F6hLKBiYb1L6PCkpUxOFyn8UK/dYmNl+wFBCksjVfmgo51i4+7fu3sjdd3P33QjjNce7e6WLoWWwVP5HniO0JjCzRoSuqAU1GWQNSeVYLAKOBDCzVoREkY/XZx0FnBmd/XQQ8K27f1HekzKy68nTV/4j66R4LG4DtgWejsbzF7n78bEFnSYpHou8kOKxGAMcbWZzgI3AVe6ec63uFI/FYOBBM7ucMLA9MBe/WJrZcMKXg0bReMx1wFYA7v4AYXymJzAf+AE4K6X95uCxEhGRapSpXU8iIpIhlChERCQpJQoREUlKiUJERJJSohARkaSUKCQjmdlGM5uRcNstybZrquH1HjGzhdFrTYtm71Z0H8PMrHV0/48l1r1T1Rij/RQdl1lm9oKZbV/O9u1ztVKq1BydHisZyczWuPu21b1tkn08Arzo7iPN7GjgdndvV4X9VTmm8vZrZo8CH7n735JsP5BQQfei6o5F8odaFJIVzGzb6Fob08xsppn9rGqsme1iZuMTvnEfFj1+tJm9Gz33aTMr7wN8PNAieu4V0b5mmdll0WPbmNlLZvZ+9Pgp0ePjzKyjmd0C1IvieCJatyb6OcLMjkuI+REzO8nMapnZbWb2XnSdgPNSOCzvEhV0M7MDo/c43czeMbO9olnKNwCnRLGcEsX+sJlNjrYtrfquyObirp+um26l3QgziWdEt2cJVQS2i9Y1IswsLWoRr4l+Dgauie7XItR+akT44N8mevz3wLWlvN4jwEnR/ZOBScD+wExgG8LM99nAfkBf4MGE5zaIfo4juv5FUUwJ2xTFeCLwaHS/NqGSZz1gEPCn6PE6wBSgeSlxrkl4f08DPaLl7YAto/tHAc9E9wcC/0x4/k3A6dH97Qn1n7aJ+/etW2bfMrKEhwiw1t3bFy2Y2VbATWZ2OLCJ8E16Z2BZwnPeAx6Otn3O3WeYWRfChWrejsqb1CZ8Ey/NbWb2J0INoLMJtYGedffvoxj+CxwGvAL8w8xuJXRXvVWB9zUauNvM6gA9gPHuvjbq7mpnZidF2zUgFPBbWOL59cxsRvT+5wL/S9j+UTNrSShRsVUZr380cLyZXRkt1wWaRfsSKZUShWSL3wA7Afu7+wYL1WHrJm7g7uOjRHIc8IiZ3QGsBP7n7qem8BpXufvIogUzO7K0jdz9IwvXvegJ3Ghmr7v7Dam8CXdfZ2bjgGOAUwgX2YFwxbGL3X1MObtY6+7tzWxrQm2jC4F7CBdrGuvuJ0YD/+PKeL4Bfd19XirxioDGKCR7NAC+ipLEEcDPrgtu4VrhX7r7g8AwwiUhJwKHmFnRmMM2ZrZniq/5FnCCmW1tZtsQuo3eMrNfAT+4++OEgoylXXd4Q9SyKc1/CMXYilonED70zy96jpntGb1mqTxc0fASYLAVl9kvKhc9MGHT1YQuuCJjgIstal5ZqDwskpQShWSLJ4COZjYTOBP4sJRtugLvm9l0wrf1u919OeGDc7iZfUDodto7lRd092mEsYvJhDGLYe4+HdgHmBx1AV0H3FjK0wuAD4oGs0t4lXBxqdc8XLoTQmKbA0wzs1mEsvFJW/xRLB8QLsrzd+Dm6L0nPm8s0LpoMJvQ8tgqim12tCySlE6PFRGRpNSiEBGRpJQoREQkKSUKERFJSolCRESSUqIQEZGklChERCQpJQoREUnq/wEmdXE6814P+gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XekOuD6KbS2Q"
      },
      "source": [
        "The Bert Classifer achieves 0.90 AUC score and 82.65% accuracy rate on the validation set. This result is 10 points better than the baseline method."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G5DW6grRmfT-"
      },
      "source": [
        "### 3.5. Train Our Model on the Entire Training Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JkMK5VqJJvSO",
        "outputId": "b45bfe68-88d4-4922-b079-9bd19930aa25",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 442
        }
      },
      "source": [
        "# Concatenate the train set and the validation set\n",
        "full_train_data = torch.utils.data.ConcatDataset([train_data, val_data])\n",
        "full_train_sampler = RandomSampler(full_train_data)\n",
        "full_train_dataloader = DataLoader(full_train_data, sampler=full_train_sampler, batch_size=32)\n",
        "\n",
        "# Train the Bert Classifier on the entire training data\n",
        "set_seed(42)\n",
        "bert_classifier, optimizer, scheduler = initialize_model(epochs=2)\n",
        "train(bert_classifier, full_train_dataloader, epochs=2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Start training...\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   1    |   20    |   0.664452   |     -      |     -     |   8.63   \n",
            "   1    |   40    |   0.587205   |     -      |     -     |   8.42   \n",
            "   1    |   60    |   0.522831   |     -      |     -     |   8.44   \n",
            "   1    |   80    |   0.476442   |     -      |     -     |   8.23   \n",
            "   1    |   100   |   0.467542   |     -      |     -     |   8.10   \n",
            "   1    |   106   |   0.483039   |     -      |     -     |   2.14   \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   2    |   20    |   0.338174   |     -      |     -     |   8.36   \n",
            "   2    |   40    |   0.296080   |     -      |     -     |   7.93   \n",
            "   2    |   60    |   0.295626   |     -      |     -     |   7.96   \n",
            "   2    |   80    |   0.277470   |     -      |     -     |   7.99   \n",
            "   2    |   100   |   0.314746   |     -      |     -     |   8.07   \n",
            "   2    |   106   |   0.293359   |     -      |     -     |   2.17   \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            "Training complete!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q89oT0n3N0m6"
      },
      "source": [
        "## 4. Predictions on Test Set"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Sqk_CPwjN_W0"
      },
      "source": [
        "### 4.1. Data Preparation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_U3K1LbDZTOU"
      },
      "source": [
        "Let's revisit out test set shortly."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MaPBmrFBO-uQ",
        "outputId": "fbb12b03-4753-480b-e71b-d9e144126723",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "test_data.sample(5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>tweet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>471</th>\n",
              "      <td>18654</td>\n",
              "      <td>Friends and family: Never fly @JetBlue.  Absol...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1971</th>\n",
              "      <td>76265</td>\n",
              "      <td>@DeltaAssist @rogerioad I never have had a pro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>672</td>\n",
              "      <td>First flight in weeks. Counting on you @Americ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2702</th>\n",
              "      <td>103263</td>\n",
              "      <td>\"@USAirways: You know that we can__t stay no m...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>135</th>\n",
              "      <td>5137</td>\n",
              "      <td>@southwestair Here at SA Airport watching the ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          id                                              tweet\n",
              "471    18654  Friends and family: Never fly @JetBlue.  Absol...\n",
              "1971   76265  @DeltaAssist @rogerioad I never have had a pro...\n",
              "23       672  First flight in weeks. Counting on you @Americ...\n",
              "2702  103263  \"@USAirways: You know that we can__t stay no m...\n",
              "135     5137  @southwestair Here at SA Airport watching the ..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AzCpJBgWZYR_"
      },
      "source": [
        "Before making predictions on the test set, we need to redo processing and encoding steps done on the training data. Fortunately, we have written the `preprocessing_for_bert` function to do that for us."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "56QTDchdOHBL",
        "outputId": "cc606108-a672-408a-f8a9-0eca70a9d749",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Run `preprocessing_for_bert` on the test set\n",
        "print('Tokenizing data...')\n",
        "test_inputs, test_masks = preprocessing_for_bert(test_data.tweet)\n",
        "\n",
        "# Create the DataLoader for our test set\n",
        "test_dataset = TensorDataset(test_inputs, test_masks)\n",
        "test_sampler = SequentialSampler(test_dataset)\n",
        "test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=32)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tokenizing data...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pYv9lSXsQCZ2"
      },
      "source": [
        "### 4.2. Predictions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YsSlCGCAajmD"
      },
      "source": [
        "There are about 300 non-negative tweets in our test set. Therefore, we will keep adjusting the decision threshold until we have about 300 non-negative tweets.\n",
        "\n",
        "The threshold we will use is 0.992, meaning that tweets with a predicted probability greater than 99.2% will be predicted positive. This value is very high compared to the default 0.5 threshold.\n",
        "\n",
        "After manually examining the test set, I find that the sentiment classification task here is even difficult for human. Therefore, a high threshold will give us safe predictions."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tGx8h7yXRkfI",
        "outputId": "e0390768-45a4-4544-ca9d-eff72311cbf5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Compute predicted probabilities on the test set\n",
        "probs = bert_predict(bert_classifier, test_dataloader)\n",
        "\n",
        "# Get predictions from the probabilities\n",
        "threshold = 0.9\n",
        "preds = np.where(probs[:, 1] > threshold, 1, 0)\n",
        "\n",
        "# Number of tweets predicted non-negative\n",
        "print(\"Number of tweets predicted non-negative: \", preds.sum())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of tweets predicted non-negative:  454\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0GMqDdsScTQb"
      },
      "source": [
        "Now we will examine 20 random tweets from our predictions. 17 of them are correct, showing that the BERT Classifier acquires about 0.85 precision rate."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aCTfCTRfWZhe",
        "outputId": "5892ecd8-fc1e-4f86-d82f-c94af30005b7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "output = test_data[preds==1]\n",
        "list(output.sample(20).tweet)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[\"@Delta @DeltaAssist Delta strikes out again. Sky lounge at the country's busiest airport closed on weekends. Dumb &amp; cheap. miss you @united\",\n",
              " '.@SouthwestAir brought back honey roasted peanuts. Is it sad that this realization may be the high point in my day?#SmallThingsInLife',\n",
              " '@DeltaAssist I emailed kana@delta and contactus.delta@delta to resolve issues two weeks ago w/o response.  advice on who to contact?',\n",
              " \"Woman With Kicked Off Flight By @AlaskaAir Because So Has #Cancer Plans to Donate Her Family's Airfare http://t.co/Uj6rispWLb\",\n",
              " \"@united (2/2) I didn't break the bag. If I didn't have to pay to check it, I wouldn't be as upset. Rather fly @AmericanAir @SouthwestAir etc\",\n",
              " \"I've flown about every airline &amp; never have I had a better experience than flying @JetBlue. Quality, service, comfort, &amp; affordability. A++\",\n",
              " '@JetBlue  Best airline to work for miss u lots #keepingitminty ',\n",
              " 'Convinced @firetweet to book a last minute trip to join me in Austin tom! Been singing the @VirginAmerica safety song ever since. Poor Eric.',\n",
              " '@AmericanAir waiting patiently to take off from #DFW to #ord http://t.co/j1oDSc6fht',\n",
              " 'Oh @JetBlue today is a sad day for B6 loyalists. I know you\\'re touting your new \"options\" but your service/no bag fees ARE what make u great',\n",
              " 'Things that are good about this flight: @Gogo and the great experience of @VirginAmerica. Not so good: the baby barf/rotten tuna smell.',\n",
              " '@USAirways @AmericanAir will miss USAir :(',\n",
              " '@altonbrown @united Time to switch to @VirginAmerica',\n",
              " 'Its never the wrong time for Chobani, @AmericanAir Admirals Club! #brokenrecord #toomanywasabipeas #lunch',\n",
              " \"On my flight, I stole my human's phone to try out @alaskaair 's new streaming IFE. It's good! Too bad she doesn't own an iPad, though.\",\n",
              " \"Can't wait for the @USAirways and @AmericanAir merger to be completed, what a hassle for the customer!\",\n",
              " \"@JetBlue I'm a broke college kid so $150 is a huge deal.\",\n",
              " \"Can't wait to fly back to the Bay Area tonight on @SouthwestAir flight 2256!!!!\",\n",
              " 'Hanging at #SFO waiting for the fog to burn for the next connection @VirginAmerica to #sxsw! #SXSW2015 #Austin',\n",
              " \"@DeltaAssist anyway I can switch flights from 1308 to one that isn't indefinitely delayed.... And get back to dc!\"]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k_OcVenocEH_"
      },
      "source": [
        "# E - Conclusion"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UMg9ZUvocF6U"
      },
      "source": [
        "By adding a simple one-hidden-layer neural network classifier on top of BERT and fine-tuning BERT, we can achieve near state-of-the-art performance, which is 10 points better than the baseline method although we only have 3,400 data points.\n",
        "\n",
        "In addition, although BERT is very large, complicated, and have millions of parameters, we only need to fine-tune it in only 2-4 epochs. That result can be achieved because BERT was trained on the huge amount and already encode a lot of information about our language. An impresive performance achieved in a short amount of time, with a small amount of data has shown why BERT is one of the most powerful NLP models available at the moment."
      ]
    }
  ]
}