riiid_transformer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "riiid-transformer.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "mount_file_id": "1hOWAP7YEJITZpJUOpPv8DSUPA6ttHvf_",
      "authorship_tag": "ABX9TyMmp6CHwK8CFHEu4SQRCEiz",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "df84cc7c15cb4a1b88e70a68aa036818": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8cd2a5459b8d4ec2bf4bc8c1edbdf3ab",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_65aab87ad9d7436cbc0572958bdc9e74",
              "IPY_MODEL_a35b7017d4264b7cbe6fab067cfe2c59"
            ]
          }
        },
        "8cd2a5459b8d4ec2bf4bc8c1edbdf3ab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "65aab87ad9d7436cbc0572958bdc9e74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9e782383c65f417faecb62a8190a8e2d",
            "_dom_classes": [],
            "description": "trn_loss - 0.5612: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_726b7c3dd6d047e9b9af04c996575e95"
          }
        },
        "a35b7017d4264b7cbe6fab067cfe2c59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d434ff4399224932804a0b26d2b076e8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [15:33&lt;00:00,  7.12it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8f97a2ea92d34de28eba6bae186b809f"
          }
        },
        "9e782383c65f417faecb62a8190a8e2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "726b7c3dd6d047e9b9af04c996575e95": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d434ff4399224932804a0b26d2b076e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8f97a2ea92d34de28eba6bae186b809f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "473ffb66bc0d416d916f21a5570f70dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_63e623ca8729418580a2fbec217269de",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7c0c9fd2c6254d028b6aa2d244d8a933",
              "IPY_MODEL_3bd813074b0a4fc596c67358b8b6562d"
            ]
          }
        },
        "63e623ca8729418580a2fbec217269de": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "7c0c9fd2c6254d028b6aa2d244d8a933": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f2e3b278ea1b423ebcedb24aef7410d0",
            "_dom_classes": [],
            "description": "val_loss - 0.5417: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_32955a66bec74c319e8c29a366ba7974"
          }
        },
        "3bd813074b0a4fc596c67358b8b6562d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e0c9ac4b120948dea71dce16902855fa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [01:15&lt;00:00, 12.59it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6b9c009ff067445ea1b8b8db94deb385"
          }
        },
        "f2e3b278ea1b423ebcedb24aef7410d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "32955a66bec74c319e8c29a366ba7974": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e0c9ac4b120948dea71dce16902855fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6b9c009ff067445ea1b8b8db94deb385": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5633ea47bf8949c98c566c20e11619f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a86258d956b847aca01a1eea2d3c6140",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b80f2471c42047b79b6d4e1317fd1544",
              "IPY_MODEL_28fd6161cfa44cc3b8cb98d715b91567"
            ]
          }
        },
        "a86258d956b847aca01a1eea2d3c6140": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "b80f2471c42047b79b6d4e1317fd1544": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cc1d542440c645c4b4d705fdd2afdfa4",
            "_dom_classes": [],
            "description": "trn_loss - 0.5411: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d4567437f0b145b7832dd07907640492"
          }
        },
        "28fd6161cfa44cc3b8cb98d715b91567": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6d7913187b2647e9a60a6de87ca55b19",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [16:53&lt;00:00,  6.56it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_18e756f6f1a14287ae4c79b96a1b887f"
          }
        },
        "cc1d542440c645c4b4d705fdd2afdfa4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d4567437f0b145b7832dd07907640492": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6d7913187b2647e9a60a6de87ca55b19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "18e756f6f1a14287ae4c79b96a1b887f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "64dfa90f814642a6bc2a214a940aa52f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ed70d94f4ef842d1b31269789d2308d6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_748c2259deef467daf5efaeba3bc5e76",
              "IPY_MODEL_81423ca9914244809e076744a9819db1"
            ]
          }
        },
        "ed70d94f4ef842d1b31269789d2308d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "748c2259deef467daf5efaeba3bc5e76": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_81a85f534c41473c896bc0baad5a02cc",
            "_dom_classes": [],
            "description": "val_loss - 0.5256: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4814cf50c5c54e28afe17d5d08d9abdd"
          }
        },
        "81423ca9914244809e076744a9819db1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_39797611ecea492b9b80a8a68a1dbe10",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [01:17&lt;00:00, 12.28it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_76a403dee600497eb77c176a1a67b9a5"
          }
        },
        "81a85f534c41473c896bc0baad5a02cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4814cf50c5c54e28afe17d5d08d9abdd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "39797611ecea492b9b80a8a68a1dbe10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "76a403dee600497eb77c176a1a67b9a5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b8d47ead000d4c468872e1f8d11b67b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a8dfaa3c6f564b4a94c4b1f4922076dc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8629e2e1e93049539cf7d6b33e2a78e9",
              "IPY_MODEL_4ffb0ccea2104ccca7858b5eb833e75a"
            ]
          }
        },
        "a8dfaa3c6f564b4a94c4b1f4922076dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "8629e2e1e93049539cf7d6b33e2a78e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_351b52d5b5df412f9dd5ec45bebee6fb",
            "_dom_classes": [],
            "description": "trn_loss - 0.5340: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0daeac05956d45faaa6c862f43adca75"
          }
        },
        "4ffb0ccea2104ccca7858b5eb833e75a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4074cb065e074d6da0ffda6346755389",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [16:50&lt;00:00,  6.57it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d1aa2522d2634e1486d746efcbb89d46"
          }
        },
        "351b52d5b5df412f9dd5ec45bebee6fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0daeac05956d45faaa6c862f43adca75": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4074cb065e074d6da0ffda6346755389": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d1aa2522d2634e1486d746efcbb89d46": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d97a665a2d4942c0b286b58f9dad980a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0a6cba75aed64e869d4919acee9baaff",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f1391614523f4d73917ff02218c17d6d",
              "IPY_MODEL_7c38f83c31bf4172866eb33e5d9fdff0"
            ]
          }
        },
        "0a6cba75aed64e869d4919acee9baaff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "f1391614523f4d73917ff02218c17d6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_954cdda3f30b417294bec65bea33438b",
            "_dom_classes": [],
            "description": "val_loss - 0.5225: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5f461a66e5e14ba8bf1cfb7100ac9f6f"
          }
        },
        "7c38f83c31bf4172866eb33e5d9fdff0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8950813b9b0f47d48a53c4202eb796f2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [01:17&lt;00:00, 12.34it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_11aef95f54c0414e89ea39c8b3aa3a23"
          }
        },
        "954cdda3f30b417294bec65bea33438b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5f461a66e5e14ba8bf1cfb7100ac9f6f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8950813b9b0f47d48a53c4202eb796f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "11aef95f54c0414e89ea39c8b3aa3a23": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7e08544db5b24a0591b2c29f2f83cbc2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2057839f46754054beb005d9499d3fea",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d115acbecbb8462e9d68df1f35a7c95a",
              "IPY_MODEL_cd88ff31d79e4335a987c1c2be2e25cd"
            ]
          }
        },
        "2057839f46754054beb005d9499d3fea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "d115acbecbb8462e9d68df1f35a7c95a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e1fc7cf4e61c4553b9ca80a23c9f4ba7",
            "_dom_classes": [],
            "description": "trn_loss - 0.5306: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6c1d9653a12a4e54924e886bc5110511"
          }
        },
        "cd88ff31d79e4335a987c1c2be2e25cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_30d6f4d93ca141358ca0e013ef6cae3b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [16:49&lt;00:00,  6.58it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_787c7cb1659e4f34a0dd4928237f924d"
          }
        },
        "e1fc7cf4e61c4553b9ca80a23c9f4ba7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6c1d9653a12a4e54924e886bc5110511": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "30d6f4d93ca141358ca0e013ef6cae3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "787c7cb1659e4f34a0dd4928237f924d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "93807d8733a145db9193b9241d8b523f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_958dfebe4566473a9ccdd5c321958606",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fedad49e61eb456c94ed3971972add64",
              "IPY_MODEL_18b876b51c4e45078b1f8e58a0d98ec4"
            ]
          }
        },
        "958dfebe4566473a9ccdd5c321958606": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "fedad49e61eb456c94ed3971972add64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_dc397bc492fd484f993a3cc7e334065f",
            "_dom_classes": [],
            "description": "val_loss - 0.5207: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e9e220fb53b045a18eef15c7c946450c"
          }
        },
        "18b876b51c4e45078b1f8e58a0d98ec4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e68519bf458d4116b3ea62249fe95bf4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [01:17&lt;00:00, 12.34it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5b2ca36c9a90486eae8b3d0fb35e2d03"
          }
        },
        "dc397bc492fd484f993a3cc7e334065f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e9e220fb53b045a18eef15c7c946450c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e68519bf458d4116b3ea62249fe95bf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5b2ca36c9a90486eae8b3d0fb35e2d03": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9dbb73cd94fd4127bde382815b1af77f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3b8b2e0e62e04833be9981384a4b4a96",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ad1bfa22b56942af97513314b1757a7c",
              "IPY_MODEL_a8066afadbdf49f5b006210507cef03f"
            ]
          }
        },
        "3b8b2e0e62e04833be9981384a4b4a96": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "ad1bfa22b56942af97513314b1757a7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_08309fbf48614fbab5cd3a8b93796298",
            "_dom_classes": [],
            "description": "trn_loss - 0.5284: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e0b04cd460a42228a72dc20b8cbad71"
          }
        },
        "a8066afadbdf49f5b006210507cef03f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d202fc6fa9d24f828a2e98a744493f55",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [16:45&lt;00:00,  6.60it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f53e1cb3e2ba4fc6aff08600a135028f"
          }
        },
        "08309fbf48614fbab5cd3a8b93796298": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e0b04cd460a42228a72dc20b8cbad71": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d202fc6fa9d24f828a2e98a744493f55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f53e1cb3e2ba4fc6aff08600a135028f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8bc8439567d84e8aa740da7da70aebc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a58533372826430c866887446caed324",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e0e2ab6e40ae4c20853aa0f937ef4e0f",
              "IPY_MODEL_ac64d71f1ff2402284064dd6a1ddc334"
            ]
          }
        },
        "a58533372826430c866887446caed324": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "e0e2ab6e40ae4c20853aa0f937ef4e0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_607d25118f054e3da33f4675884e831b",
            "_dom_classes": [],
            "description": "val_loss - 0.5189: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_877f23d85a7a4f618854932a980a25ed"
          }
        },
        "ac64d71f1ff2402284064dd6a1ddc334": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_58a4820ff6384e9b8386790c97608430",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [01:17&lt;00:00, 12.34it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6fef47633b9347eebd1f3d2b4a86c578"
          }
        },
        "607d25118f054e3da33f4675884e831b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "877f23d85a7a4f618854932a980a25ed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "58a4820ff6384e9b8386790c97608430": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6fef47633b9347eebd1f3d2b4a86c578": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ab47eb143dcf4a22b2b5e5767a0c6f17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fc9a7acd627e4f3b8479ba9c51c036e3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ecfcfff08f6946ad99c8d31d522c4f28",
              "IPY_MODEL_0ea33d9760ba4902ad001ab40c09d889"
            ]
          }
        },
        "fc9a7acd627e4f3b8479ba9c51c036e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "ecfcfff08f6946ad99c8d31d522c4f28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cc81de2270f3422bbfe6636671820d9f",
            "_dom_classes": [],
            "description": "trn_loss - 0.5262: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_32325be2b3774d3e800a965501a0a8dc"
          }
        },
        "0ea33d9760ba4902ad001ab40c09d889": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0e85cb960e0c4606aea3e34ab7055982",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [16:45&lt;00:00,  6.61it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3c9f62d01b73467d8ab9e7ae0b132d88"
          }
        },
        "cc81de2270f3422bbfe6636671820d9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "32325be2b3774d3e800a965501a0a8dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0e85cb960e0c4606aea3e34ab7055982": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3c9f62d01b73467d8ab9e7ae0b132d88": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9ec375f237a240e781998269b6395abc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1350bee07c0b4b0a9e01bba9c120bb45",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fc0e40812257448f89288dc6ce5d0f02",
              "IPY_MODEL_0973eb052a1444ae90ef01b713d7e417"
            ]
          }
        },
        "1350bee07c0b4b0a9e01bba9c120bb45": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "fc0e40812257448f89288dc6ce5d0f02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ca3e3706229045ec8a222948550d9aa2",
            "_dom_classes": [],
            "description": "val_loss - 0.5180: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_74493df8dd55443fa0377ffea6c53bed"
          }
        },
        "0973eb052a1444ae90ef01b713d7e417": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a81111a74a06434e8a1266b01b0f25c9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [01:16&lt;00:00, 12.35it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4e1b66fa1f7b4ccbaa42e23e9d36b2d3"
          }
        },
        "ca3e3706229045ec8a222948550d9aa2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "74493df8dd55443fa0377ffea6c53bed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a81111a74a06434e8a1266b01b0f25c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4e1b66fa1f7b4ccbaa42e23e9d36b2d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "18ef3d0dd4ff48c2a811e3457052f982": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8b89751502134a719a01ca5f8d9bcf03",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_85297db351b444ba956b85295a985974",
              "IPY_MODEL_1da526007a254f22977ebd3dd293d359"
            ]
          }
        },
        "8b89751502134a719a01ca5f8d9bcf03": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "85297db351b444ba956b85295a985974": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_922f8d69728f4fd3bb7a78e4cfe744fe",
            "_dom_classes": [],
            "description": "trn_loss - 0.5244: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_03f9313a50ae43b293f1ca325aa281b1"
          }
        },
        "1da526007a254f22977ebd3dd293d359": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_13a60bc1bdfb484ba5156323c547bae5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [16:42&lt;00:00,  6.62it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_49b26633b4e14512a54848c4eeed53da"
          }
        },
        "922f8d69728f4fd3bb7a78e4cfe744fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "03f9313a50ae43b293f1ca325aa281b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "13a60bc1bdfb484ba5156323c547bae5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "49b26633b4e14512a54848c4eeed53da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ccaaecf15ebc424aa14465f8b824bfcb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b0a1dbea033d4fc3b8c8ef9756d4340e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4b632a37d4c44e88ae9cd3cecc06704d",
              "IPY_MODEL_8d8667b0e885487c9a19d4af76b0e3a4"
            ]
          }
        },
        "b0a1dbea033d4fc3b8c8ef9756d4340e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "4b632a37d4c44e88ae9cd3cecc06704d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_25c241b7fc764bcfb2a0c2f8805652b4",
            "_dom_classes": [],
            "description": "val_loss - 0.5173: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b3a3d6b94c7b49c984e17207b1b443d4"
          }
        },
        "8d8667b0e885487c9a19d4af76b0e3a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b05db9b1b82a49b68b468d86f83edefe",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [01:16&lt;00:00, 12.43it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c20fdd5e18e849658105db5a6f2c4d35"
          }
        },
        "25c241b7fc764bcfb2a0c2f8805652b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b3a3d6b94c7b49c984e17207b1b443d4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b05db9b1b82a49b68b468d86f83edefe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c20fdd5e18e849658105db5a6f2c4d35": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6519bbce4ab44f66957783b18f2b620a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b8f514d2a6d4432790f57cfecd177b27",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e2eec45395fa4ee592cee7d44d31ddd2",
              "IPY_MODEL_93011fdd31da4960bb2b4a23c6fe16aa"
            ]
          }
        },
        "b8f514d2a6d4432790f57cfecd177b27": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "e2eec45395fa4ee592cee7d44d31ddd2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3259289a58744f00b7812bd828d6c80a",
            "_dom_classes": [],
            "description": "trn_loss - 0.5229: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cfe767fb0c284e5e85a20d76ed889816"
          }
        },
        "93011fdd31da4960bb2b4a23c6fe16aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f13d5ccc16664e91955fa6feaa1eb47b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [16:41&lt;00:00,  6.63it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_797d355953f84092a5ff58030473ce6c"
          }
        },
        "3259289a58744f00b7812bd828d6c80a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cfe767fb0c284e5e85a20d76ed889816": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f13d5ccc16664e91955fa6feaa1eb47b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "797d355953f84092a5ff58030473ce6c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2c618ff1773a48e7a90b5a72276ed61a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5bd58ccd9ef94846ab55fd5016de4484",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a16ef1024a6746aab9765fb857a3393d",
              "IPY_MODEL_1e66d54bb5974786b1e748a449d3ef82"
            ]
          }
        },
        "5bd58ccd9ef94846ab55fd5016de4484": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "a16ef1024a6746aab9765fb857a3393d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bbc5292245334a7ab6c9980530ca0574",
            "_dom_classes": [],
            "description": "val_loss - 0.5168: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_42425a08a94d4aa19dcc50bf5f2d40d3"
          }
        },
        "1e66d54bb5974786b1e748a449d3ef82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5c7e7e97f77c4b3abe62fcc6084e1fe4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [01:16&lt;00:00, 12.40it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a82b12d43c5f4f788b05372c8284bba0"
          }
        },
        "bbc5292245334a7ab6c9980530ca0574": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "42425a08a94d4aa19dcc50bf5f2d40d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5c7e7e97f77c4b3abe62fcc6084e1fe4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a82b12d43c5f4f788b05372c8284bba0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b57c885e5527452c93bdce02ed271736": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1fe7e69d181e40adb49893ebc383bc21",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8eab1b941a4a4aeeba4527fc32ee4cf8",
              "IPY_MODEL_eae4cc7618714ad6b60c7e1fa204a3c1"
            ]
          }
        },
        "1fe7e69d181e40adb49893ebc383bc21": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "8eab1b941a4a4aeeba4527fc32ee4cf8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_41dde5caf17d43e198ffab849856c801",
            "_dom_classes": [],
            "description": "trn_loss - 0.5212: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e8eead0049e44079d0c262b74f9b677"
          }
        },
        "eae4cc7618714ad6b60c7e1fa204a3c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_94b6048cad064ff3a173b193f70675d0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [16:41&lt;00:00,  6.63it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f058c3dafa73478d8266d71c61cef939"
          }
        },
        "41dde5caf17d43e198ffab849856c801": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e8eead0049e44079d0c262b74f9b677": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "94b6048cad064ff3a173b193f70675d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f058c3dafa73478d8266d71c61cef939": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "efd169582d1a4ef3aebcd6ce7d4a1c56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7fa3a75698dd46619445845995638d29",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e41e1fc358cf488bbc6d799d4e3f45f2",
              "IPY_MODEL_27595a5587e64b23b621fe7c7375db9f"
            ]
          }
        },
        "7fa3a75698dd46619445845995638d29": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "e41e1fc358cf488bbc6d799d4e3f45f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1fd31b4755ef48a2be529a0bcdf30865",
            "_dom_classes": [],
            "description": "val_loss - 0.5154: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_613f122146ce402b89bc0e793f50c7c3"
          }
        },
        "27595a5587e64b23b621fe7c7375db9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ae1514384d884fe99541a1556d32ee4e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [01:16&lt;00:00, 12.41it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a96705dbaa2d48a098353f8c845aa63e"
          }
        },
        "1fd31b4755ef48a2be529a0bcdf30865": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "613f122146ce402b89bc0e793f50c7c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ae1514384d884fe99541a1556d32ee4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a96705dbaa2d48a098353f8c845aa63e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d4c74d600f68455d9c16cf2f2b6572a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bc1e78b35e5b4edc9f7a831be9607cc5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f0c78d186afa4c36875453acceaf841f",
              "IPY_MODEL_5a8eb97172674d1db8245db3548e3616"
            ]
          }
        },
        "bc1e78b35e5b4edc9f7a831be9607cc5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "f0c78d186afa4c36875453acceaf841f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_88852086665d46158d33c1ce4857b321",
            "_dom_classes": [],
            "description": "trn_loss - 0.5196: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6a85791520314394b55a61e0739e1bf9"
          }
        },
        "5a8eb97172674d1db8245db3548e3616": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_66a22d4a828b4517a973396a40ac6367",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [32:09&lt;00:00,  3.44it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7de635702d5440beb0851fd8eb0a4320"
          }
        },
        "88852086665d46158d33c1ce4857b321": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6a85791520314394b55a61e0739e1bf9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "66a22d4a828b4517a973396a40ac6367": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7de635702d5440beb0851fd8eb0a4320": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d9f13eafbebf464e9e9f32f357dbfade": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_79c139a4052a4d329203049244fb584c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d09a67112d444435931d22c8368c6579",
              "IPY_MODEL_b8e0cf5cb2464ba48517f78e6f294197"
            ]
          }
        },
        "79c139a4052a4d329203049244fb584c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "d09a67112d444435931d22c8368c6579": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b394d83a302c4ca5902d454b1ab9cfc0",
            "_dom_classes": [],
            "description": "val_loss - 0.5149: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3c93b9eb3bad48e2a8d8c8389881e7b0"
          }
        },
        "b8e0cf5cb2464ba48517f78e6f294197": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b34b3e4282584d5d8cf22dfb4e7fc8bc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [16:41&lt;00:00,  1.05s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2138f1ea38f54c6898220b3d20fda7b9"
          }
        },
        "b394d83a302c4ca5902d454b1ab9cfc0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3c93b9eb3bad48e2a8d8c8389881e7b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b34b3e4282584d5d8cf22dfb4e7fc8bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2138f1ea38f54c6898220b3d20fda7b9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1d065d84ec1e4c5ba5b566ecca396d5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ecbfd200a6e74105b8f1a9411d67e1d4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_77898901c5c04a2b857c5acfd56e23f3",
              "IPY_MODEL_1f53eb2889b747cb80b7e61285ea2ea7"
            ]
          }
        },
        "ecbfd200a6e74105b8f1a9411d67e1d4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "77898901c5c04a2b857c5acfd56e23f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_fe93afa91f2443db8f20238b49250c78",
            "_dom_classes": [],
            "description": "trn_loss - 0.5181: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9532e6859bca4b3bb240477d1a386014"
          }
        },
        "1f53eb2889b747cb80b7e61285ea2ea7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9133ed1a74494cae8988af6a88a6da62",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [15:24&lt;00:00,  7.18it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_05cfa69143f048b69133a28384a9e2ad"
          }
        },
        "fe93afa91f2443db8f20238b49250c78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9532e6859bca4b3bb240477d1a386014": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9133ed1a74494cae8988af6a88a6da62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "05cfa69143f048b69133a28384a9e2ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1509ecd840454564aa2e904550d74e89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_67afd9a8ec274e36b6d73e1dee2af7af",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3b6331cd0bb84f179edc693827c28a0c",
              "IPY_MODEL_3f2e3f858f6644c7bcf75e611b69647a"
            ]
          }
        },
        "67afd9a8ec274e36b6d73e1dee2af7af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "3b6331cd0bb84f179edc693827c28a0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_712de39fc28b482481a9961940f523a4",
            "_dom_classes": [],
            "description": "val_loss - 0.5149: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e488599981c04ba488c0551f382a19c4"
          }
        },
        "3f2e3f858f6644c7bcf75e611b69647a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_cf494adc90e24298b6cc650a12037fe8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [01:11&lt;00:00, 13.32it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_16494f4cdffa45b48ba7ae05f645ee8d"
          }
        },
        "712de39fc28b482481a9961940f523a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e488599981c04ba488c0551f382a19c4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cf494adc90e24298b6cc650a12037fe8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "16494f4cdffa45b48ba7ae05f645ee8d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a55ea250187b4f59b7929fb0727ba258": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2037e2dc30cc42479e152f570e89d132",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_44824f2cd5cc42e48a4568ff7d54e92f",
              "IPY_MODEL_582a7ecacc3b42078b10f1f682294474"
            ]
          }
        },
        "2037e2dc30cc42479e152f570e89d132": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "44824f2cd5cc42e48a4568ff7d54e92f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f465867c37a9459e90b272a58535f4f9",
            "_dom_classes": [],
            "description": "trn_loss - 0.5166: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_904ba60891234c2fa0cec710920944b3"
          }
        },
        "582a7ecacc3b42078b10f1f682294474": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1d9094decfd34c559d20233f88a6a8e8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [16:43&lt;00:00,  6.62it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_caf29b258f384d5c9eab60e1e779ab5a"
          }
        },
        "f465867c37a9459e90b272a58535f4f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "904ba60891234c2fa0cec710920944b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1d9094decfd34c559d20233f88a6a8e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "caf29b258f384d5c9eab60e1e779ab5a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "101464caa8484e2a9c4aca479a8e2a65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_acbdff72abc84fbaafbbe1eef4130c06",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_76f50cdfb551492cbcd984db172b4a76",
              "IPY_MODEL_804b2180e3794ada8dd9681a4a917ab2"
            ]
          }
        },
        "acbdff72abc84fbaafbbe1eef4130c06": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "76f50cdfb551492cbcd984db172b4a76": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_73b80cc30a734f68a74d973d1b071bff",
            "_dom_classes": [],
            "description": "val_loss - 0.5148: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6b5736e5c67441ad884d897dae8e6820"
          }
        },
        "804b2180e3794ada8dd9681a4a917ab2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_84c034317f7e4a6782e8aa6c46cc4020",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [01:17&lt;00:00, 12.28it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d49cdf26c1f7447cbb1982970b2f97d5"
          }
        },
        "73b80cc30a734f68a74d973d1b071bff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6b5736e5c67441ad884d897dae8e6820": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "84c034317f7e4a6782e8aa6c46cc4020": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d49cdf26c1f7447cbb1982970b2f97d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "53971220d6cd4bba8fc56aea264a8dbc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ba09fccf1b8d4b97b496d5eb9bd8c057",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1d7afe011cb04d448df48db56006fabe",
              "IPY_MODEL_1161ce97641441f1b68cd53a18529005"
            ]
          }
        },
        "ba09fccf1b8d4b97b496d5eb9bd8c057": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "1d7afe011cb04d448df48db56006fabe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_942a9b14b8cf44b99cc7411db3a34690",
            "_dom_classes": [],
            "description": "trn_loss - 0.5150: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_69ec20cd5b584253a30984463f2092a7"
          }
        },
        "1161ce97641441f1b68cd53a18529005": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a26bb8f3ba03406c840e0726cc2a4a65",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [16:43&lt;00:00,  6.62it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_63bd86e0244844eeba38ea8b7548b26d"
          }
        },
        "942a9b14b8cf44b99cc7411db3a34690": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "69ec20cd5b584253a30984463f2092a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a26bb8f3ba03406c840e0726cc2a4a65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "63bd86e0244844eeba38ea8b7548b26d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a0cbb986aa7d42389add69fab0e18937": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_782fab00eb074740ab96d329735b4373",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dfd8c94343944dd3a6819f8b6340a1f1",
              "IPY_MODEL_1ef3adef5efc47fd8c1be63d530f9127"
            ]
          }
        },
        "782fab00eb074740ab96d329735b4373": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "dfd8c94343944dd3a6819f8b6340a1f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_002dcae14957488ea1bba489bb86635c",
            "_dom_classes": [],
            "description": "val_loss - 0.5150: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a56662f45e8148adb9e956684c99fdd0"
          }
        },
        "1ef3adef5efc47fd8c1be63d530f9127": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_976ec2ccb3744aa1b6d0c68c06571391",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [01:17&lt;00:00, 12.33it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_70aa75fe3df945efa8c851b1e6abe824"
          }
        },
        "002dcae14957488ea1bba489bb86635c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a56662f45e8148adb9e956684c99fdd0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "976ec2ccb3744aa1b6d0c68c06571391": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "70aa75fe3df945efa8c851b1e6abe824": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ec7525f3906e4ca8850ecbace87e0dfe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3270d641b02b4726933162253a80427e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4c0360e1c149420fa6efe800ed3fb106",
              "IPY_MODEL_1786bc1e9bea4e5b9d3d73d2a56aa4ee"
            ]
          }
        },
        "3270d641b02b4726933162253a80427e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "4c0360e1c149420fa6efe800ed3fb106": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9b793107a8cd40f6b83679b1874100bb",
            "_dom_classes": [],
            "description": "trn_loss - 0.5132: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3df13214a39a4f62835fc4c6c6322d3c"
          }
        },
        "1786bc1e9bea4e5b9d3d73d2a56aa4ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f44d73ca57184d819742673784bf3d19",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [32:11&lt;00:00,  3.44it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5715a2d871704d6cba7f8e45ca933fc7"
          }
        },
        "9b793107a8cd40f6b83679b1874100bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3df13214a39a4f62835fc4c6c6322d3c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f44d73ca57184d819742673784bf3d19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5715a2d871704d6cba7f8e45ca933fc7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4e02369b94b548af9cb10c6473bfb616": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8b5b2b399ef147dbb0636f1fe1a537e9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_463de4c99cb1412c800da584c2fb63ca",
              "IPY_MODEL_a78e20209f1a4f2d8971c623a943049e"
            ]
          }
        },
        "8b5b2b399ef147dbb0636f1fe1a537e9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "463de4c99cb1412c800da584c2fb63ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9fc1c0bbe2d54f79bafa50861c4da31f",
            "_dom_classes": [],
            "description": "val_loss - 0.5159: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3ffe2bd239dc4b1d802af45a193d1f88"
          }
        },
        "a78e20209f1a4f2d8971c623a943049e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_13dd00d1fd774db9a6177710958d8e56",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [16:45&lt;00:00,  1.06s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fdfddd25e161418a9cac39e30ca36ab6"
          }
        },
        "9fc1c0bbe2d54f79bafa50861c4da31f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3ffe2bd239dc4b1d802af45a193d1f88": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "13dd00d1fd774db9a6177710958d8e56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fdfddd25e161418a9cac39e30ca36ab6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "12f1bc6fa1fc427abd216de8539da1f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3f86b65af0e94c18a7db4859f3ac4082",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_76152c60c3fe4440a8497510b0305c1c",
              "IPY_MODEL_a887db49709d48558d7381215dd7e2a0"
            ]
          }
        },
        "3f86b65af0e94c18a7db4859f3ac4082": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "76152c60c3fe4440a8497510b0305c1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4df05383921b404a92f5c4d2b1b07a5a",
            "_dom_classes": [],
            "description": "trn_loss - 0.5102: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d213521d5e154771a2521ba715bf4544"
          }
        },
        "a887db49709d48558d7381215dd7e2a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_13c6ad736acd455ba471b24b197b66b4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [15:28&lt;00:00,  7.15it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_84e94fe45b7e4d429d64275de40268ef"
          }
        },
        "4df05383921b404a92f5c4d2b1b07a5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d213521d5e154771a2521ba715bf4544": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "13c6ad736acd455ba471b24b197b66b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "84e94fe45b7e4d429d64275de40268ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b836b3b015f14c2abd7f8b279a823ea9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b0500040a7b84927b5b2360b175bf224",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c5a281682f0b4b20943b80d4e5ea743c",
              "IPY_MODEL_a8a7cf017d57425281d0d5d90c3e8391"
            ]
          }
        },
        "b0500040a7b84927b5b2360b175bf224": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "c5a281682f0b4b20943b80d4e5ea743c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1137ab97d3cd47288d92a6b50097ba4d",
            "_dom_classes": [],
            "description": "val_loss - 0.5144: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8d1ce78aac9746fea32c1fdf5975d909"
          }
        },
        "a8a7cf017d57425281d0d5d90c3e8391": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_11dc6f2c2fca4fb7a12342e9a1784348",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [01:12&lt;00:00, 13.16it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dbea74a8f70b44a19bdac2487e5f6e1f"
          }
        },
        "1137ab97d3cd47288d92a6b50097ba4d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8d1ce78aac9746fea32c1fdf5975d909": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "11dc6f2c2fca4fb7a12342e9a1784348": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dbea74a8f70b44a19bdac2487e5f6e1f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1f856acceb304e1ab4a7b5475ecf20c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_26eae43436ec46e3931109b2c1e7f915",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7fd08036b71c46f5b2f4f466484a12c2",
              "IPY_MODEL_bbb282986ab14d3eac6595be86ae3b17"
            ]
          }
        },
        "26eae43436ec46e3931109b2c1e7f915": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "7fd08036b71c46f5b2f4f466484a12c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1c0f4f33b0c247d396587f7a89330ebc",
            "_dom_classes": [],
            "description": "trn_loss - 0.5086: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2a31c5d85aef4442a333826b9f009885"
          }
        },
        "bbb282986ab14d3eac6595be86ae3b17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_02f3ce638a1742939b346fe1363d9aa3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6642/6642 [15:29&lt;00:00,  7.15it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c61a818da11c4cfa9f4bec4eed814058"
          }
        },
        "1c0f4f33b0c247d396587f7a89330ebc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2a31c5d85aef4442a333826b9f009885": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "02f3ce638a1742939b346fe1363d9aa3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c61a818da11c4cfa9f4bec4eed814058": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fca05926b73b4e128166792ec2d6b1d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ea027488b83d41c48013d8d305974f29",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9f07715a942043b6a0bc1a319440ebf0",
              "IPY_MODEL_891a5c0098d14fc28cc03f1e4de60d12"
            ]
          }
        },
        "ea027488b83d41c48013d8d305974f29": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "9f07715a942043b6a0bc1a319440ebf0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2bb101a82e5a44b3a2934c79e6820060",
            "_dom_classes": [],
            "description": "val_loss - 0.5161: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 951,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 951,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5fa8aed958664c98a94bb962ce59a1ee"
          }
        },
        "891a5c0098d14fc28cc03f1e4de60d12": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fa0b244b33bf4685a5c80705171e5eb1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 951/951 [01:13&lt;00:00, 12.92it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_607ceab609db4fe2bda86bc4f083b575"
          }
        },
        "2bb101a82e5a44b3a2934c79e6820060": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5fa8aed958664c98a94bb962ce59a1ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fa0b244b33bf4685a5c80705171e5eb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "607ceab609db4fe2bda86bc4f083b575": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b8e99e932a9345bebb42c653c69e62f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0c1a1fa9c56740a28c1f337d92ebe902",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_acb91b4772db46639b92bd7d67e54bc2",
              "IPY_MODEL_f3a4c9b3d76a4fbeaaa6266f600766dc"
            ]
          }
        },
        "0c1a1fa9c56740a28c1f337d92ebe902": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "700px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "acb91b4772db46639b92bd7d67e54bc2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1ff4eb6c8d6c49cbbb0b658eeae5b96d",
            "_dom_classes": [],
            "description": "trn_loss - 0.5080:   2%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 6642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 109,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_80d1f472fe174717a009feb8b6e486c2"
          }
        },
        "f3a4c9b3d76a4fbeaaa6266f600766dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9011e41760304c39be30187606e91db4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 109/6642 [00:15&lt;15:30,  7.02it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8a78bcdb72a94ad4a39239db1a42f8e3"
          }
        },
        "1ff4eb6c8d6c49cbbb0b658eeae5b96d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "80d1f472fe174717a009feb8b6e486c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9011e41760304c39be30187606e91db4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8a78bcdb72a94ad4a39239db1a42f8e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bagherig/riiid/blob/main/riiid_transformer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pUvAipNhK_a1",
        "outputId": "ae477059-3798-442d-9373-5e3ba0f5e7f8"
      },
      "source": [
        "gpu_info = !nvidia-smi\r\n",
        "gpu_info = '\\n'.join(gpu_info)\r\n",
        "if gpu_info.find('failed') >= 0:\r\n",
        "  print('Select the Runtime > \"Change runtime type\" menu to enable a GPU accelerator, ')\r\n",
        "  print('and then re-execute this cell.')\r\n",
        "else:\r\n",
        "  print(gpu_info)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fri Jan 15 03:27:07 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.27.04    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla V100-SXM2...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   38C    P0    44W / 300W |      0MiB / 16130MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vrfx2prPLbUC",
        "outputId": "9a1c153b-9a52-430c-8827-ce68afca9d7d"
      },
      "source": [
        "from psutil import virtual_memory\r\n",
        "ram_gb = virtual_memory().total / 1e9\r\n",
        "print('Your runtime has {:.1f} gigabytes of available RAM\\n'.format(ram_gb))\r\n",
        "\r\n",
        "if ram_gb < 20:\r\n",
        "  print('To enable a high-RAM runtime, select the Runtime > \"Change runtime type\"')\r\n",
        "  print('menu, and then select High-RAM in the Runtime shape dropdown. Then, ')\r\n",
        "  print('re-execute this cell.')\r\n",
        "else:\r\n",
        "  print('You are using a high-RAM runtime!')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Your runtime has 27.4 gigabytes of available RAM\n",
            "\n",
            "You are using a high-RAM runtime!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8pRZ01_lCNvF"
      },
      "source": [
        "# Global Variables"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CWr3RLMn-xh2"
      },
      "source": [
        "# %% [code]\r\n",
        "\r\n",
        "COMPILE = False #@param {type:\"boolean\"}\r\n",
        "VERSION = 'v1' #@param ['v1', 'v2']\r\n",
        "TPU = True\r\n",
        "\r\n",
        "DRIVE_DIR = '/content/drive/MyDrive/Colab Notebooks/datasets' + \\\r\n",
        "             '/riiid-answer-correctness-prediction/'\r\n",
        "HOME_DIR = './'\r\n",
        "DATA_DIR = DRIVE_DIR + 'data/riiid-test-answer-prediction/'\r\n",
        "DRIVE_PARQUETS_DIR = DRIVE_DIR + 'data/parquets/'\r\n",
        "PARQUETS_DIR = HOME_DIR + 'parquets/'\r\n",
        "MODELS_DIR = DRIVE_DIR + 'models/'\r\n",
        "\r\n",
        "OUT_DIR = DRIVE_DIR + 'temp/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A0_U89HBCJUU"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w5H2P6BOB8Zh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "be99d43e-8b48-4b3b-86b2-302ab15e4669"
      },
      "source": [
        "# import riiideducation \r\n",
        "\r\n",
        "import os\r\n",
        "import gc\r\n",
        "import sys\r\n",
        "import math\r\n",
        "import copy\r\n",
        "import random\r\n",
        "import psutil\r\n",
        "import warnings\r\n",
        "from typing import Optional, Any\r\n",
        "\r\n",
        "import numpy as np\r\n",
        "import pandas as pd\r\n",
        "import torch\r\n",
        "import torch.nn as nn\r\n",
        "import torch.nn.functional as F\r\n",
        "from torch import Tensor\r\n",
        "from torch.nn.modules.module import Module\r\n",
        "from torch.nn.modules.activation import MultiheadAttention\r\n",
        "from torch.nn.modules.container import ModuleList\r\n",
        "from torch.nn.modules.dropout import Dropout\r\n",
        "from torch.nn.modules.linear import Linear\r\n",
        "from torch.nn.modules.normalization import LayerNorm\r\n",
        "from sklearn.metrics import roc_auc_score\r\n",
        "from tqdm.notebook import tqdm\r\n",
        "\r\n",
        "warnings.filterwarnings('ignore')\r\n",
        "gc.enable()\r\n",
        "\r\n",
        "if not os.path.exists(PARQUETS_DIR):\r\n",
        "    sys.path.append(DRIVE_PARQUETS_DIR)\r\n",
        "    zip_path = DRIVE_PARQUETS_DIR + 'parquets.zip'\r\n",
        "    !cp '{zip_path}' $HOME_DIR\r\n",
        "    !unzip -q 'parquets.zip'\r\n",
        "    !rm 'parquets.zip'\r\n",
        "\r\n",
        "TPU = 'COLAB_TPU_ADDR' in os.environ\r\n",
        "if TPU:\r\n",
        "    !pip install cloud-tpu-client==0.10 https://storage.googleapis.com/tpu-pytorch/wheels/torch_xla-1.7-cp36-cp36m-linux_x86_64.whl\r\n",
        "    import torch_xla.core.xla_model as xm\r\n",
        "    import torch_xla.distributed.parallel_loader as pl\r\n",
        "    import torch_xla.distributed.xla_multiprocessing as xmp\r\n",
        "    \r\n",
        "print(torch.__version__)\r\n",
        "\r\n",
        "def seed_everything(seed):\r\n",
        "    random.seed(seed)\r\n",
        "    np.random.seed(seed)\r\n",
        "    os.environ['PYTHONHASHSEED'] = str(seed)\r\n",
        "    torch.manual_seed(seed)\r\n",
        "    torch.cuda.manual_seed(seed)\r\n",
        "    torch.cuda.manual_seed_all(seed)# if you are using multi-GPU.\r\n",
        "    torch.backends.cudnn.benchmark = False\r\n",
        "    torch.backends.cudnn.deterministic = True\r\n",
        "    \r\n",
        "RANDOM_SEED = 1\r\n",
        "seed_everything(RANDOM_SEED)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.7.0+cu101\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pr8EVxJpCW8E"
      },
      "source": [
        "TARGET = 'answered_correctly'\r\n",
        "KEY_FEATURE = 'user_id'\r\n",
        "FEATURES = [\r\n",
        "    'content_id',\r\n",
        "    'timestamp',\r\n",
        "    'prior_question_elapsed_time',\r\n",
        "    'prior_question_had_explanation',\r\n",
        "    # 'task_container_id',\r\n",
        "    'part',\r\n",
        "    'tag1',\r\n",
        "    'tag2',\r\n",
        "    # 'tag3',\r\n",
        "    # 'tag4',\r\n",
        "    # 'tag5',\r\n",
        "    # 'tag6',\r\n",
        "]\r\n",
        "\r\n",
        "DTYPES = {\r\n",
        "    'content_id': int,\r\n",
        "    'timestamp': int,\r\n",
        "    'prior_question_elapsed_time': int,\r\n",
        "    'prior_question_had_explanation': int,\r\n",
        "    # 'task_container_id': int,\r\n",
        "    'part': int,\r\n",
        "    'tag1': int,\r\n",
        "    'tag2': int,\r\n",
        "    # 'tag3': int,\r\n",
        "    # 'tag4': int,\r\n",
        "    # 'tag5': int,\r\n",
        "    # 'tag6': int,\r\n",
        "}\r\n",
        "\r\n",
        "# For Inference\r\n",
        "ADDED_FEATURES = [\r\n",
        "    'part',\r\n",
        "    'tag1',\r\n",
        "    'tag2',\r\n",
        "    # 'tag3',\r\n",
        "    # 'tag4',\r\n",
        "    # 'tag5',\r\n",
        "    # 'tag6',\r\n",
        "]\r\n",
        "\r\n",
        "DEFAULT_VALUES = {\r\n",
        "    'prior_question_had_explanation': False,\r\n",
        "    'prior_question_elapsed_time': 26000,\r\n",
        "    'timestamp': 0,\r\n",
        "    'part': 0,\r\n",
        "    'tag1': -1,\r\n",
        "    'tag2': -1,\r\n",
        "    # 'tag3': -1,\r\n",
        "    # 'tag4': -1,\r\n",
        "    # 'tag5': -1,\r\n",
        "    # 'tag6': -1,\r\n",
        "}\r\n",
        "\r\n",
        "TIME_SCALES = {\r\n",
        "    'prior_question_elapsed_time': 1000,\r\n",
        "    'timestamp': 1000 * 5,\r\n",
        "}\r\n",
        "SUBMISSION_COLUMNS = [\r\n",
        "    'row_id',\r\n",
        "    TARGET\r\n",
        "]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1zjqePrN3L8V"
      },
      "source": [
        "# Helper Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1hYdsN6WCuyt"
      },
      "source": [
        "def prepare_data():\r\n",
        "    print(\"Reading datasets...\")\r\n",
        "    dt = pd.read_parquet(PARQUETS_DIR + 'train.parquet')\r\n",
        "    questions = pd.read_parquet(PARQUETS_DIR + 'questions.parquet')\r\n",
        "\r\n",
        "    print(\"Merging datasets...\")\r\n",
        "    dt = dt[(dt['content_type_id'] == 0) & (dt['answered_correctly'] != -1)] \\\r\n",
        "        .drop(columns=['content_type_id'])\r\n",
        "    questions.index.name = 'content_id'\r\n",
        "    questions = questions.reset_index()\r\n",
        "    dt = dt.merge(questions, on='content_id', how='left').reset_index(drop=True)\r\n",
        "\r\n",
        "    print('Processing timestamps...')\r\n",
        "    if 'timestamp' in dt.columns:\r\n",
        "        timestamps_raw = dt['timestamp'].values.astype(int)\r\n",
        "        dt['timestamp'] = dt[[KEY_FEATURE, 'timestamp']]\\\r\n",
        "            .groupby(KEY_FEATURE)['timestamp'].diff()\r\n",
        "        while 0 in dt['timestamp'].values:\r\n",
        "            dt['timestamp'][dt['timestamp'] == 0] = \\\r\n",
        "                dt['timestamp'].shift()[dt['timestamp'] == 0]\r\n",
        "        dt['timestamp'] = dt['timestamp'].fillna(\r\n",
        "            DEFAULT_VALUES['timestamp']).astype(int)\r\n",
        "        dt['timestamp_raw'] = timestamps_raw\r\n",
        "\r\n",
        "    print('Writing dataset to .parquet files...')\r\n",
        "    dt.to_parquet(PARQUETS_DIR + 'train_merged.parquet')\r\n",
        "\r\n",
        "    return dt\r\n",
        "\r\n",
        "\r\n",
        "def load_data(filename, cols):\r\n",
        "    return pd.read_parquet(PARQUETS_DIR + filename, columns=cols)\r\n",
        "\r\n",
        "\r\n",
        "def preprocess(dt: pd.DataFrame):\r\n",
        "    dt[\"content_id\"] += 1\r\n",
        "    if 'prior_question_elapsed_time' in dt.columns:\r\n",
        "        dt[\"prior_question_elapsed_time\"] = \\\r\n",
        "            dt[\"prior_question_elapsed_time\"].fillna(\r\n",
        "                DEFAULT_VALUES['prior_question_elapsed_time'])\r\n",
        "        dt[\"prior_question_elapsed_time\"] = \\\r\n",
        "            np.ceil(dt[\"prior_question_elapsed_time\"] / \r\n",
        "                    TIME_SCALES['prior_question_elapsed_time']).astype(np.int32)\r\n",
        "    if 'prior_question_had_explanation' in dt.columns:\r\n",
        "        dt['prior_question_had_explanation'].fillna(\r\n",
        "            DEFAULT_VALUES['prior_question_had_explanation'], inplace=True)\r\n",
        "    if 'timestamp' in dt.columns:\r\n",
        "        dt['timestamp'] = np.ceil(\r\n",
        "            dt['timestamp'] / TIME_SCALES['timestamp']).astype(np.int32)\r\n",
        "\r\n",
        "    for col in [\"task_container_id\", \r\n",
        "                \"tag1\", \"tag2\", \"tag3\", \"tag4\", \"tag5\", \"tag6\"]:\r\n",
        "        if col in dt.columns:\r\n",
        "            dt[col] += 1\r\n",
        "\r\n",
        "    return dt\r\n",
        "\r\n",
        "\r\n",
        "def preprocess_test(dt: np.ndarray, all_cols):\r\n",
        "    dt[:, all_cols[\"content_id\"]] += 1\r\n",
        "    if 'prior_question_had_explanation' in all_cols:\r\n",
        "        time_nans = pd.isnull(dt[:, all_cols[\"prior_question_elapsed_time\"]])\r\n",
        "        dt[time_nans, all_cols[\"prior_question_elapsed_time\"]] = \\\r\n",
        "            DEFAULT_VALUES['prior_question_elapsed_time']\r\n",
        "        dt[:, all_cols[\"prior_question_elapsed_time\"]] = \\\r\n",
        "            np.ceil(dt[:, all_cols[\"prior_question_elapsed_time\"]] / \r\n",
        "                    TIME_SCALES['prior_question_elapsed_time']).astype(np.int32)\r\n",
        "    if 'prior_question_had_explanation' in all_cols:\r\n",
        "        explanation_nans = pd.isnull(\r\n",
        "            dt[:, all_cols[\"prior_question_had_explanation\"]])\r\n",
        "        dt[explanation_nans, all_cols[\"prior_question_had_explanation\"]] = \\\r\n",
        "            DEFAULT_VALUES['prior_question_had_explanation']\r\n",
        "    for col in [\"task_container_id\", \r\n",
        "                \"tag1\", \"tag2\", \"tag3\", \"tag4\", \"tag5\", \"tag6\"]:\r\n",
        "        if col in all_cols:\r\n",
        "            dt[:, all_cols[col]] += 1\r\n",
        "    return dt\r\n",
        "\r\n",
        "\r\n",
        "def split_train_valid(dt, val_fraction):\r\n",
        "    if val_fraction == 1:\r\n",
        "        return None, dt\r\n",
        "    val_size = 0\r\n",
        "    trn_size = 0\r\n",
        "    val_uids = []\r\n",
        "    val_fraction = val_fraction / (1 - val_fraction)\r\n",
        "    n_samples_per_user = dt.groupby(KEY_FEATURE)[\r\n",
        "        TARGET].count().sort_values().reset_index().values.tolist()\r\n",
        "    while n_samples_per_user:\r\n",
        "        uid, nsamples = n_samples_per_user.pop()\r\n",
        "        if trn_size * val_fraction >= val_size:\r\n",
        "            val_uids.append(uid)\r\n",
        "            val_size += nsamples\r\n",
        "        else:\r\n",
        "            trn_size += nsamples\r\n",
        "\r\n",
        "    val = dt[dt[KEY_FEATURE].isin(val_uids)]\r\n",
        "    trn = dt.drop(val.index)\r\n",
        "    return trn, val\r\n",
        "\r\n",
        "\r\n",
        "def pad_batch(x, window_size, pad_value=0):\r\n",
        "    pad_dims = (window_size - x.shape[0], 0)\r\n",
        "    shape = (pad_dims,) + tuple(\r\n",
        "        (0, 0) for i in range(len(x.shape) - 1))\r\n",
        "    return np.pad(x, shape, constant_values=pad_value)\r\n",
        "\r\n",
        "\r\n",
        "def create_scheduler(optim):\r\n",
        "    sched = None\r\n",
        "    if SCHED_TYPE == 'plateau':\r\n",
        "        sched = torch.optim.lr_scheduler.ReduceLROnPlateau(optim, 'max',\r\n",
        "                                                           patience=0,\r\n",
        "                                                           factor=0.5,\r\n",
        "                                                           threshold=5e-6,\r\n",
        "                                                           min_lr=1e-6)\r\n",
        "    return sched\r\n",
        "\r\n",
        "\r\n",
        "def create_optimizer(estimator, lr):\r\n",
        "    return torch.optim.Adam(estimator.parameters(), lr=lr)\r\n",
        "\r\n",
        "\r\n",
        "def create_model(model_type, **params):\r\n",
        "    return model_type(**params).to(DEVICE)\r\n",
        "\r\n",
        "\r\n",
        "def save_model(estimator, optim, sched=None, val_score:float=0, epoch=0):\r\n",
        "    checkpoint = {'model_params': estimator.params,\r\n",
        "                  'model_state_dict': estimator.state_dict(),\r\n",
        "                  'optimizer_state_dict': optim.state_dict(),\r\n",
        "                  'learning_rate': optim.param_groups[0]['lr'],\r\n",
        "                  'val_score': val_score}\r\n",
        "    if sched is not None:\r\n",
        "        checkpoint = {**checkpoint,\r\n",
        "                      'sheduler_state_dict': sched.state_dict(),\r\n",
        "                      'epoch': epoch}\r\n",
        "    torch.save(checkpoint, OUT_DIR + MODEL_FILENAME)\r\n",
        "\r\n",
        "\r\n",
        "def load_model(model_type, for_training=False):\r\n",
        "    if os.path.exists(OUT_DIR + MODEL_FILENAME):\r\n",
        "        filepath = OUT_DIR + MODEL_FILENAME\r\n",
        "    else:\r\n",
        "        filepath = MODELS_DIR + f'transformer/{VERSION}/{MODEL_FILENAME}'\r\n",
        "\r\n",
        "    print(f'Loading model from {filepath}')\r\n",
        "    checkpoint = torch.load(filepath, map_location=DEVICE)\r\n",
        "\r\n",
        "    estimator = create_model(model_type, **checkpoint['model_params'])\r\n",
        "    estimator.load_state_dict(checkpoint['model_state_dict'])\r\n",
        "\r\n",
        "    optim = None\r\n",
        "    sched = None\r\n",
        "    if for_training:\r\n",
        "        optim = create_optimizer(estimator, lr=checkpoint['learning_rate'])\r\n",
        "        optim.load_state_dict(checkpoint['optimizer_state_dict'])\r\n",
        "\r\n",
        "        if 'sheduler_state_dict' in checkpoint:\r\n",
        "            sched = create_scheduler(optim)\r\n",
        "            sched.load_state_dict(checkpoint['sheduler_state_dict'])\r\n",
        "\r\n",
        "    return estimator, optim, sched, checkpoint"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2l3Dn_VHy_aZ"
      },
      "source": [
        "#Transformer Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NehUzt_Jeuq1"
      },
      "source": [
        "class PositionalEncoding(nn.Module):\r\n",
        "    def __init__(self, d_model, max_len):\r\n",
        "        super().__init__()\r\n",
        "        pe = torch.zeros(max_len, d_model)\r\n",
        "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\r\n",
        "        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (\r\n",
        "                -math.log(10000.0) / d_model))\r\n",
        "        pe[:, 0::2] = torch.sin(position * div_term)\r\n",
        "        pe[:, 1::2] = torch.cos(position * div_term)\r\n",
        "        self.register_buffer('pe', pe)\r\n",
        "\r\n",
        "    def forward(self, pos):\r\n",
        "        return self.pe[pos, :]\r\n",
        "\r\n",
        "\r\n",
        "class TransformerEncoder(Module):\r\n",
        "    __constants__ = ['norm']\r\n",
        "\r\n",
        "    def __init__(self, encoder_layer, num_layers, norm=None):\r\n",
        "        super(TransformerEncoder, self).__init__()\r\n",
        "        self.layers = _get_clones(encoder_layer, num_layers)\r\n",
        "        self.num_layers = num_layers\r\n",
        "        self.norm = norm\r\n",
        "\r\n",
        "    def forward(self, src: Tensor, mask: Optional[Tensor] = None) -> Tensor:\r\n",
        "        output = src\r\n",
        "\r\n",
        "        for mod in self.layers:\r\n",
        "            output = mod(output, mask=mask)\r\n",
        "\r\n",
        "        if self.norm is not None:\r\n",
        "            output = self.norm(output)\r\n",
        "\r\n",
        "        return output\r\n",
        "\r\n",
        "\r\n",
        "class TransformerDecoder(Module):\r\n",
        "    __constants__ = ['norm']\r\n",
        "\r\n",
        "    def __init__(self, decoder_layer, num_layers, norm=None):\r\n",
        "        super(TransformerDecoder, self).__init__()\r\n",
        "        self.layers = _get_clones(decoder_layer, num_layers)\r\n",
        "        self.num_layers = num_layers\r\n",
        "        self.norm = norm\r\n",
        "\r\n",
        "    def forward(self, tgt: Tensor, memory: Tensor, mask) -> Tensor:\r\n",
        "        output = tgt\r\n",
        "\r\n",
        "        for mod in self.layers:\r\n",
        "            output = mod(output, memory, mask=mask)\r\n",
        "\r\n",
        "        if self.norm is not None:\r\n",
        "            output = self.norm(output)\r\n",
        "\r\n",
        "        return output\r\n",
        "\r\n",
        "\r\n",
        "class TransformerEncoderLayer(Module):\r\n",
        "    def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1,\r\n",
        "                 activation=\"relu\"):\r\n",
        "        super(TransformerEncoderLayer, self).__init__()\r\n",
        "        self.self_attn = MultiheadAttention(d_model, nhead, dropout=dropout)\r\n",
        "        # Implementation of Feedforward model\r\n",
        "        self.linear1 = Linear(d_model, dim_feedforward)\r\n",
        "        self.dropout = Dropout(dropout)\r\n",
        "        self.linear2 = Linear(dim_feedforward, d_model)\r\n",
        "\r\n",
        "        self.norm1 = LayerNorm(d_model)\r\n",
        "        self.norm2 = LayerNorm(d_model)\r\n",
        "        self.dropout1 = Dropout(dropout)\r\n",
        "        self.dropout2 = Dropout(dropout)\r\n",
        "\r\n",
        "        self.activation = _get_activation_fn(activation)\r\n",
        "\r\n",
        "    def __setstate__(self, state):\r\n",
        "        if 'activation' not in state:\r\n",
        "            state['activation'] = F.relu\r\n",
        "        super(TransformerEncoderLayer, self).__setstate__(state)\r\n",
        "\r\n",
        "    def forward(self, src: Tensor, mask: Optional[Tensor] = None) -> Tensor:\r\n",
        "        src2 = self.self_attn(src, src, src, attn_mask=mask)[0]\r\n",
        "        src = src + self.dropout1(src2)\r\n",
        "        src = self.norm1(src)\r\n",
        "        src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))\r\n",
        "        src = src + self.dropout2(src2)\r\n",
        "        src = self.norm2(src)\r\n",
        "        return src\r\n",
        "\r\n",
        "\r\n",
        "class TransformerDecoderLayer(Module):\r\n",
        "    def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1,\r\n",
        "                 activation=\"relu\"):\r\n",
        "        super(TransformerDecoderLayer, self).__init__()\r\n",
        "        self.self_attn = MultiheadAttention(d_model, nhead, dropout=dropout)\r\n",
        "        self.multihead_attn = MultiheadAttention(d_model, nhead,\r\n",
        "                                                 dropout=dropout)\r\n",
        "        # Implementation of Feedforward model\r\n",
        "        self.linear1 = Linear(d_model, dim_feedforward)\r\n",
        "        self.dropout = Dropout(dropout)\r\n",
        "        self.linear2 = Linear(dim_feedforward, d_model)\r\n",
        "\r\n",
        "        self.norm1 = LayerNorm(d_model)\r\n",
        "        self.norm2 = LayerNorm(d_model)\r\n",
        "        self.norm3 = LayerNorm(d_model)\r\n",
        "        self.dropout1 = Dropout(dropout)\r\n",
        "        self.dropout2 = Dropout(dropout)\r\n",
        "        self.dropout3 = Dropout(dropout)\r\n",
        "\r\n",
        "        self.activation = _get_activation_fn(activation)\r\n",
        "\r\n",
        "    def __setstate__(self, state):\r\n",
        "        if 'activation' not in state:\r\n",
        "            state['activation'] = F.relu\r\n",
        "        super(TransformerDecoderLayer, self).__setstate__(state)\r\n",
        "\r\n",
        "    def forward(self, tgt: Tensor, memory: Tensor, \r\n",
        "                mask: Optional[Tensor] = None) -> Tensor:\r\n",
        "        tgt2 = self.self_attn(tgt, tgt, tgt, attn_mask=mask )[0]\r\n",
        "        tgt = tgt + self.dropout1(tgt2)\r\n",
        "        tgt = self.norm1(tgt)\r\n",
        "        tgt2 = self.multihead_attn(tgt, memory, memory, attn_mask=mask)[0]\r\n",
        "        tgt = tgt + self.dropout2(tgt2)\r\n",
        "        tgt = self.norm2(tgt)\r\n",
        "        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))\r\n",
        "        tgt = tgt + self.dropout3(tgt2)\r\n",
        "        tgt = self.norm3(tgt)\r\n",
        "        return tgt\r\n",
        "\r\n",
        "\r\n",
        "def _get_clones(module, N):\r\n",
        "    return ModuleList([copy.deepcopy(module) for i in range(N)])\r\n",
        "\r\n",
        "\r\n",
        "def _get_activation_fn(activation):\r\n",
        "    if activation == \"relu\":\r\n",
        "        return F.relu\r\n",
        "    elif activation == \"gelu\":\r\n",
        "        return F.gelu\r\n",
        "    elif activation == \"tanh\":\r\n",
        "        return F.tanh\r\n",
        "    elif activation == \"sigmoid\":\r\n",
        "        return F.sigmoid\r\n",
        "\r\n",
        "    raise RuntimeError(\r\n",
        "        \"activation should be relu/gelu/tanh/sigmoid, not {}\".format(activation))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ug2Ys4AmzCon"
      },
      "source": [
        "class TransformerModel(nn.Module):\r\n",
        "    def __init__(self, **params):\r\n",
        "        print(params)\r\n",
        "        super().__init__()\r\n",
        "        self.params = params.copy()\r\n",
        "        self.d_model = params['d_model']\r\n",
        "        self.nhead = params['nhead']\r\n",
        "        self.seq_len = params.pop('seq_len') - 1\r\n",
        "        n_enc_layers = params.pop('num_encoder_layers')\r\n",
        "        n_dec_layers = params.pop('num_decoder_layers')\r\n",
        "        dropout_rate = params['dropout']\r\n",
        "\r\n",
        "        self.enc_norm = nn.LayerNorm(self.d_model)\r\n",
        "        self.dec_norm = nn.LayerNorm(self.d_model)\r\n",
        "\r\n",
        "        encoder_layer = TransformerEncoderLayer(**params)\r\n",
        "        self.encoder = TransformerEncoder(encoder_layer=encoder_layer,\r\n",
        "                                          num_layers=n_enc_layers,\r\n",
        "                                          norm=self.enc_norm)\r\n",
        "        decoder_layer = TransformerDecoderLayer(**params)\r\n",
        "        self.decoder = TransformerDecoder(decoder_layer=decoder_layer,\r\n",
        "                                          num_layers=n_dec_layers,\r\n",
        "                                          norm=self.dec_norm)\r\n",
        "        self.max_exercise = 13523\r\n",
        "        self.max_part = 7\r\n",
        "        self.max_explanation = 1\r\n",
        "        self.max_tag = 188\r\n",
        "        self.max_target = 2\r\n",
        "        self.max_time = 300  # seconds\r\n",
        "        self.max_timestamp = int((1 * 24 * 60 * 60 * 1000) /\r\n",
        "                                 TIME_SCALES['timestamp'])  # 1 day\r\n",
        "\r\n",
        "        self.position_embeddings = PositionalEncoding(self.d_model,\r\n",
        "                                                      self.seq_len)\r\n",
        "        self.exercise_embeddings = \\\r\n",
        "            nn.Embedding(num_embeddings=self.max_exercise + 1,\r\n",
        "                         embedding_dim=self.d_model,\r\n",
        "                         scale_grad_by_freq=True)\r\n",
        "        self.part_embeddings = \\\r\n",
        "            nn.Embedding(num_embeddings=self.max_part + 1,\r\n",
        "                         embedding_dim=self.d_model,\r\n",
        "                         scale_grad_by_freq=True)\r\n",
        "        self.explanation_embeddings = \\\r\n",
        "            nn.Embedding(num_embeddings=self.max_explanation + 1,\r\n",
        "                         embedding_dim=self.d_model,\r\n",
        "                         scale_grad_by_freq=True)\r\n",
        "        self.timestamp_embeddings = (\r\n",
        "            nn.Embedding(num_embeddings=self.max_timestamp + 1,\r\n",
        "                         embedding_dim=self.d_model))\r\n",
        "        self.elapsed_time_embeddings = (\r\n",
        "            nn.Embedding(num_embeddings=self.max_time + 1,\r\n",
        "                         embedding_dim=self.d_model))\r\n",
        "        self.tag1_embeddings = \\\r\n",
        "            nn.Embedding(num_embeddings=self.max_tag + 1,\r\n",
        "                         embedding_dim=self.d_model,\r\n",
        "                         scale_grad_by_freq=True)\r\n",
        "        self.tag2_embeddings = \\\r\n",
        "            nn.Embedding(num_embeddings=self.max_tag + 1,\r\n",
        "                         embedding_dim=self.d_model,\r\n",
        "                         scale_grad_by_freq=True)\r\n",
        "        self.target_embeddings = \\\r\n",
        "            nn.Embedding(num_embeddings=self.max_target + 1,\r\n",
        "                         embedding_dim=self.d_model,\r\n",
        "                         scale_grad_by_freq=True)\r\n",
        "\r\n",
        "        self.linear = nn.Linear(self.d_model, 1)\r\n",
        "        self.sigmoid = nn.Sigmoid()\r\n",
        "        self.dropout = nn.Dropout(dropout_rate)\r\n",
        "\r\n",
        "        self.device = DEVICE\r\n",
        "        self.future_mask = self.get_future_mask().to(self.device)\r\n",
        "        self.positions = torch.arange(0, self.seq_len,\r\n",
        "                                      device=self.device).unsqueeze(0)\r\n",
        "        self.init_weights()\r\n",
        "\r\n",
        "    def get_future_mask(self):\r\n",
        "        return torch.triu(torch.ones(self.seq_len, self.seq_len) == 1, \r\n",
        "                          diagonal=1)\r\n",
        "\r\n",
        "    def init_weights(self):\r\n",
        "        initrange = 0.1\r\n",
        "        self.exercise_embeddings.weight.data.uniform_(-initrange, initrange)\r\n",
        "        self.part_embeddings.weight.data.uniform_(-initrange, initrange)\r\n",
        "        self.explanation_embeddings.weight.data.uniform_(-initrange, initrange)\r\n",
        "        self.elapsed_time_embeddings.weight.data.uniform_(-initrange,\r\n",
        "                                                          initrange)\r\n",
        "        self.timestamp_embeddings.weight.data.uniform_(-initrange, initrange)\r\n",
        "        self.tag1_embeddings.weight.data.uniform_(-initrange, initrange)\r\n",
        "        self.tag2_embeddings.weight.data.uniform_(-initrange, initrange)\r\n",
        "        self.target_embeddings.weight.data.uniform_(-initrange, initrange)\r\n",
        "        self.linear.bias.data.zero_()\r\n",
        "        self.linear.weight.data.uniform_(-initrange, initrange)\r\n",
        "\r\n",
        "    def forward(self, inputs: dict):\r\n",
        "        (content_ids,\r\n",
        "         timestamps,\r\n",
        "         elapsed_times,\r\n",
        "         explanations,\r\n",
        "         parts,\r\n",
        "         tags1, tags2,\r\n",
        "         targets) = inputs.values()  # (N, S)\r\n",
        "\r\n",
        "        content_ids[content_ids > self.max_exercise] = 0\r\n",
        "        parts[parts > self.max_part] = 0\r\n",
        "        timestamps[timestamps > self.max_timestamp] = self.max_timestamp\r\n",
        "        elapsed_times[elapsed_times > self.max_time] = self.max_time\r\n",
        "\r\n",
        "        embedded_inp = (self.exercise_embeddings(content_ids)\r\n",
        "                        + self.part_embeddings(parts)\r\n",
        "                        + self.explanation_embeddings(explanations)\r\n",
        "                        + self.elapsed_time_embeddings(elapsed_times)\r\n",
        "                        + self.timestamp_embeddings(timestamps)\r\n",
        "                        + self.tag1_embeddings(tags1)\r\n",
        "                        + self.tag2_embeddings(tags2) * 0.8\r\n",
        "                        + self.target_embeddings(targets)\r\n",
        "                        ).transpose(0, 1) * np.sqrt(self.d_model) # (S, N, E)\r\n",
        "\r\n",
        "        pos = self.positions.repeat(content_ids.shape[0], 1)\r\n",
        "        embedded_pos = self.position_embeddings(pos).transpose(0, 1)\r\n",
        "\r\n",
        "        output = self.encoder(src=(embedded_inp + embedded_pos),\r\n",
        "                              mask=self.future_mask)  # (S, N, E)\r\n",
        "\r\n",
        "        output = (self.target_embeddings(targets).transpose(0, 1)\r\n",
        "                  + output) * np.sqrt(self.d_model)\r\n",
        "        tgt = (self.exercise_embeddings(content_ids)\r\n",
        "               + self.target_embeddings(targets)\r\n",
        "               ).transpose(0, 1) * np.sqrt(self.d_model)\r\n",
        "        output = self.decoder(tgt=(tgt + embedded_pos),\r\n",
        "                              memory=(output + embedded_pos),\r\n",
        "                              mask=self.future_mask)  # (S, N, E)\r\n",
        "\r\n",
        "        output = self.linear(output.transpose(1, 0)).squeeze(-1)\r\n",
        "        output = self.sigmoid(output)\r\n",
        "\r\n",
        "        return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v6bS8TrLx5GK"
      },
      "source": [
        "# Training Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k94CLZe84gcs"
      },
      "source": [
        "class Riiid(torch.utils.data.Dataset):\n",
        "    def __init__(self, dt, seq_len, pad_value=0, is_training=True):\n",
        "        super().__init__()\n",
        "        self.seq_len = seq_len\n",
        "        self.min_seq_len = 20\n",
        "        self.pad_value = pad_value\n",
        "        self.is_training = is_training\n",
        "        self.dcols = {col: i for i, col in\n",
        "                      enumerate(FEATURES + [TARGET])}\n",
        "        self.users_data = dt.groupby(KEY_FEATURE).apply(\n",
        "            lambda r: r[FEATURES + [TARGET]].values).values\n",
        "        if is_training:\n",
        "            samples = [[gi] * (min(1000, len(g)) // self.seq_len + 1)\n",
        "                        for gi, g in enumerate(self.users_data) \n",
        "                        if len(g) > self.min_seq_len]\n",
        "        else:\n",
        "            samples = [[gi] * (len(g) // self.seq_len + 1)\n",
        "                        for gi, g in enumerate(self.users_data) \n",
        "                        if len(g) > self.min_seq_len]            \n",
        "        self.samples = [(gi, len(self.users_data[gi]) - gii * self.seq_len) \n",
        "                        for g in samples for gii, gi in enumerate(g) \n",
        "                        if len(self.users_data[gi]) - gii * self.seq_len > \n",
        "                        self.min_seq_len]\n",
        "        random.shuffle(self.samples)\n",
        "        \n",
        "    def __len__(self):\n",
        "        return len(self.samples)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        uid, end_idx = self.samples[idx]\n",
        "        inputs = self.users_data[uid]\n",
        "        if self.is_training:\n",
        "            end_idx = np.random.randint(self.min_seq_len, len(inputs))\n",
        "        start_idx = max(0, end_idx - self.seq_len)\n",
        "        inputs = inputs[start_idx: end_idx]\n",
        "        targets = inputs[:, self.dcols[TARGET]] + 1\n",
        "        inputs = pad_batch(inputs, self.seq_len, self.pad_value)\n",
        "        targets = pad_batch(targets, self.seq_len, self.pad_value)\n",
        "        inputs = np.c_[inputs[1:], targets[:-1]]\n",
        "        return inputs\n",
        "\n",
        "\n",
        "def collate_fn(batch):\n",
        "    return np.array(batch).transpose(2, 0, 1)\n",
        "\n",
        "\n",
        "def train_epoch(estimator, trn_iterator, optim, criterion,\n",
        "                device=\"cpu\"):\n",
        "    target_idx = -1\n",
        "    label_idx = -2\n",
        "    estimator.train()\n",
        "\n",
        "    num_corrects = 0\n",
        "    loss_sum = 0\n",
        "    batch_count = 0\n",
        "    sample_count = 0\n",
        "\n",
        "    if TPU:\n",
        "        trn_iterator = pl.ParallelLoader(trn_iterator,\n",
        "                                         [device]).per_device_loader(device)\n",
        "    tbar = tqdm(trn_iterator, ncols=700)\n",
        "    for batch in tbar:\n",
        "        inputs = {}\n",
        "        for i, feat in enumerate(FEATURES):\n",
        "            tens = torch.Tensor(batch[i]).to(device)\n",
        "            if DTYPES[feat] is int:\n",
        "                inputs[feat] = tens.long()\n",
        "            elif DTYPES[feat] is float:\n",
        "                inputs[feat] = tens.float()\n",
        "            elif DTYPES[feat] is bool:\n",
        "                inputs[feat] = tens.bool()\n",
        "        inputs['targets'] = torch.Tensor(batch[target_idx]).to(\n",
        "            device).long()\n",
        "        labels = torch.Tensor(batch[label_idx]).to(\n",
        "            device).float()\n",
        "        optim.zero_grad()\n",
        "        output = estimator(inputs=inputs)\n",
        "        mask = inputs['targets'] != PAD_VALUE\n",
        "        output = torch.masked_select(output, mask)\n",
        "        labels = torch.masked_select(labels, mask)\n",
        "        loss = criterion(output, labels)\n",
        "        loss.backward()\n",
        "        if TPU:\n",
        "            xm.optimizer_step(optim)\n",
        "        else:\n",
        "            optim.step()\n",
        "\n",
        "        loss_sum += loss.item()\n",
        "        pred = (output >= 0.5).long()\n",
        "        num_corrects += (pred == labels).sum().item()\n",
        "        batch_count += 1\n",
        "        sample_count += len(labels)\n",
        "\n",
        "        tbar.set_description(\n",
        "            'trn_loss - {:.4f}'.format(loss_sum / batch_count))\n",
        "\n",
        "    acc = num_corrects / sample_count\n",
        "    loss = loss_sum / batch_count\n",
        "\n",
        "    return loss, acc\n",
        "\n",
        "\n",
        "def val_epoch(estimator, val_iterator, criterion, device=\"cpu\"):\n",
        "    target_idx = -1\n",
        "    label_idx = -2\n",
        "    estimator.eval()\n",
        "\n",
        "    loss_sum = 0\n",
        "    batch_count = 0\n",
        "    num_corrects = 0\n",
        "    sample_count = 0\n",
        "    truth = np.empty(0)\n",
        "    outs = np.empty(0)\n",
        "    truth_single = np.empty(0)\n",
        "    outs_single = np.empty(0)\n",
        "    \n",
        "    if TPU:\n",
        "        val_iterator = pl.ParallelLoader(val_iterator,\n",
        "                                         [device]).per_device_loader(device)\n",
        "    tbar = tqdm(val_iterator, ncols=700)\n",
        "    for batch in tbar:\n",
        "        inputs = {}\n",
        "        for i, feat in enumerate(FEATURES):\n",
        "            tens = torch.Tensor(batch[i]).to(device)\n",
        "            if DTYPES[feat] is int:\n",
        "                inputs[feat] = tens.long()\n",
        "            elif DTYPES[feat] is float:\n",
        "                inputs[feat] = tens.float()\n",
        "            elif DTYPES[feat] is bool:\n",
        "                inputs[feat] = tens.bool()\n",
        "        inputs['targets'] = torch.Tensor(batch[target_idx]).to(device).long()\n",
        "        labels = torch.Tensor(batch[label_idx]).to(device).float()\n",
        "\n",
        "        with torch.no_grad():\n",
        "            output = estimator(inputs=inputs)\n",
        "        output_single = output[:, -1]\n",
        "        labels_single = labels[:, -1]\n",
        "        mask = inputs['targets'] != PAD_VALUE\n",
        "        output = torch.masked_select(output, mask)\n",
        "        labels = torch.masked_select(labels, mask)\n",
        "        loss = criterion(output, labels)\n",
        "        loss_sum += loss.item()\n",
        "        batch_count += 1\n",
        "\n",
        "        pred = (output >= 0.5).long()\n",
        "        num_corrects += (pred == labels).sum().item()\n",
        "        sample_count += len(labels)\n",
        "        truth = np.r_[truth, labels.view(-1).data.cpu().numpy()]\n",
        "        outs = np.r_[outs, output.view(-1).data.cpu().numpy()]\n",
        "        truth_single = np.r_[truth_single, \n",
        "                             labels_single.view(-1).data.cpu().numpy()]\n",
        "        outs_single = np.r_[outs_single, \n",
        "                            output_single.view(-1).data.cpu().numpy()]\n",
        "        \n",
        "        tbar.set_description(\n",
        "            'val_loss - {:.4f}'.format(loss_sum / batch_count))\n",
        "\n",
        "    loss = loss_sum / batch_count\n",
        "    acc = num_corrects / sample_count\n",
        "    auc = roc_auc_score(truth, outs)\n",
        "    single_auc = roc_auc_score(truth_single, outs_single)\n",
        "    return loss, acc, auc, single_auc\n",
        "\n",
        "\n",
        "def train_transformer(estimator_type, estimator_params, epochs, batch_size,\n",
        "                      device, early_stopping, eps, nworkers, cont, debug,\n",
        "                      debug_size, data_size, valid_size):\n",
        "    train = load_data('train_merged.parquet', \n",
        "                      cols=[KEY_FEATURE] + FEATURES + [TARGET])\n",
        "    print('Using Columns -', list(train.columns))\n",
        "    print('total size:', train.shape)\n",
        "    data_size = int(len(train) * (debug_size if debug else data_size))\n",
        "    train = train.tail(data_size)\n",
        "    print('data size:', train.shape,\n",
        "          '- num users:', train['user_id'].nunique())\n",
        "    train = preprocess(train)\n",
        "    train, valid = split_train_valid(train, valid_size)\n",
        "    print('train size:', train.shape,\n",
        "          '- num users:', train['user_id'].nunique())\n",
        "    print('valid size:', valid.shape,\n",
        "          '- num users:', valid['user_id'].nunique())\n",
        "    gc.collect()\n",
        "\n",
        "    \n",
        "    last_auc = 0\n",
        "    last_epoch = 0\n",
        "    if cont and not debug:\n",
        "        estimator, optim, sched, cp = \\\n",
        "            load_model(MODEL, for_training=True)\n",
        "        last_auc = cp['val_score']\n",
        "        last_epoch = cp['epoch']\n",
        "        print('Previous Validation AUC:', last_auc)\n",
        "    else:\n",
        "        estimator = create_model(estimator_type, **estimator_params)\n",
        "        optim = create_optimizer(estimator, lr=LEARNING_RATE)\n",
        "        sched = create_scheduler(optim)\n",
        "\n",
        "    trn_dataset = Riiid(dt=train, seq_len=SEQ_LEN, pad_value=PAD_VALUE, \n",
        "                        is_training=True)\n",
        "    val_dataset = Riiid(dt=valid, seq_len=SEQ_LEN, pad_value=PAD_VALUE, \n",
        "                        is_training=False)\n",
        "\n",
        "    if TPU:\n",
        "        trn_sampler = torch.utils.data.distributed.DistributedSampler(\n",
        "            trn_dataset,\n",
        "            num_replicas=xm.xrt_world_size(),\n",
        "            rank=xm.get_ordinal(),\n",
        "            shuffle=True)\n",
        "\n",
        "        val_sampler = torch.utils.data.distributed.DistributedSampler(\n",
        "            val_dataset,\n",
        "            num_replicas=xm.xrt_world_size(),\n",
        "            rank=xm.get_ordinal(),\n",
        "            shuffle=False)\n",
        "\n",
        "    trn_dataloader = torch.utils.data.DataLoader(\n",
        "        dataset=trn_dataset,\n",
        "        batch_size=batch_size,\n",
        "        collate_fn=collate_fn,\n",
        "        num_workers=nworkers,\n",
        "        sampler=trn_sampler if TPU else None,\n",
        "        shuffle=False if TPU else True)\n",
        "    val_dataloader = torch.utils.data.DataLoader(\n",
        "        dataset=val_dataset,\n",
        "        batch_size=batch_size,\n",
        "        collate_fn=collate_fn,\n",
        "        num_workers=nworkers,\n",
        "        sampler=val_sampler if TPU else None,\n",
        "        shuffle=False)\n",
        "\n",
        "    criterion = nn.BCELoss().to(device)\n",
        "    over_fit = 0\n",
        "    seed_everything(RANDOM_SEED)\n",
        "    for epoch in range(last_epoch, epochs):\n",
        "        if SCHED_TYPE == 'lambda':\n",
        "            for g in optim.param_groups:\n",
        "                g['lr'] = ((estimator.d_model ** -0.5) * \n",
        "                           min((epoch + 1) ** -0.5, \n",
        "                               (epoch + 1) * (WARMUP_STEPS ** -1.5)))\n",
        "        print(f'\\nEpoch {epoch + 1} - Learning rate:',\n",
        "              optim.param_groups[0]['lr'])\n",
        "        trn_loss, trn_acc = train_epoch(estimator, trn_dataloader,\n",
        "                                        optim, criterion, device)\n",
        "        print(\"  Training -- loss: {:.6f} - acc: {:.6f}\" \\\n",
        "              .format(trn_loss, trn_acc))\n",
        "\n",
        "        val_loss, val_acc, val_auc, val_single_auc = \\\n",
        "            val_epoch(estimator, val_dataloader, criterion, device)\n",
        "        color = '\\033[91m' if val_single_auc > last_auc and epoch != 0 else ''\n",
        "        print(color + \"  Validation -- loss: {:.6f} - acc: {:.6f} \"\\\n",
        "              .format(val_loss, val_acc) + \n",
        "              \"- auc: {:.6f} - custom_auc: {:.6f}\"\\\n",
        "              .format(val_auc, val_single_auc) + '\\033[0m')\n",
        "        if SCHED_TYPE == 'plateau':\n",
        "            sched.step(val_auc)\n",
        "\n",
        "        if val_single_auc > last_auc + eps:\n",
        "            last_auc = val_single_auc\n",
        "            over_fit = 0\n",
        "            save_model(estimator, optim, sched, val_single_auc, epoch)\n",
        "        else:\n",
        "            over_fit += 1\n",
        "\n",
        "        if over_fit >= early_stopping:\n",
        "            print(\"early stop epoch \", epoch + 1)\n",
        "            break\n",
        "\n",
        "    return estimator\n",
        "\n",
        "\n",
        "def tpu_map_fn(index, flags):\n",
        "    train_transformer(flags)\n",
        "\n",
        "\n",
        "# ========================== TEST =======================================\n",
        "\n",
        "\n",
        "class RiiidTest(torch.utils.data.Dataset):\n",
        "    def __init__(self, dt, queries, seq_len, pad_value=0, local=False):\n",
        "        self.data = dt\n",
        "        self.dcols = {col: i for i, col in enumerate(\n",
        "            [KEY_FEATURE] + FEATURES + [TARGET, 'timestamp_raw'])}\n",
        "\n",
        "        self.queries = queries.copy()\n",
        "        self.groups = None\n",
        "        if local:\n",
        "            self.groups = queries.groupby(KEY_FEATURE) \\\n",
        "                .apply(lambda r: r.values).values.tolist()\n",
        "        self.seq_len = seq_len\n",
        "        self.pad_value = pad_value\n",
        "        self.is_local = local\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.queries)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        if self.is_local:\n",
        "            random.shuffle(self.groups)\n",
        "            query = self.groups[0][[0]]\n",
        "            query_label = query[0, self.dcols[TARGET]]\n",
        "            query = np.delete(query, self.dcols[TARGET], axis=1)\n",
        "            self.groups[0] = self.groups[0][1:]\n",
        "            if self.groups[0].shape[0] == 0:\n",
        "                self.groups = self.groups[1:]\n",
        "        else:\n",
        "            query = self.queries[[idx]]\n",
        "        uid = query[0, self.dcols[KEY_FEATURE]]\n",
        "\n",
        "        if uid in self.data:\n",
        "            inputs = self.data[uid][-self.seq_len + 1:]\n",
        "            labels = inputs[:, self.dcols[TARGET]]\n",
        "            last_timestamp = inputs[-1, self.dcols['timestamp_raw']]\n",
        "            curr_timestamp = query[0, self.dcols['timestamp']]\n",
        "            diff_timestamp = np.ceil((curr_timestamp - last_timestamp) / \n",
        "                                     TIME_SCALES['timestamp'])\n",
        "            query[0, self.dcols['timestamp']] = diff_timestamp\n",
        "            query = np.delete(query, self.dcols[KEY_FEATURE], axis=1)\n",
        "            inputs = np.delete(inputs, [self.dcols[KEY_FEATURE],\n",
        "                                        self.dcols[TARGET],\n",
        "                                        self.dcols['timestamp_raw']], axis=1)\n",
        "            inputs = np.r_[inputs, query]\n",
        "        else:\n",
        "            query[0, self.dcols['timestamp']] = DEFAULT_VALUES['timestamp']\n",
        "            query = np.delete(query, self.dcols[KEY_FEATURE], axis=1)\n",
        "            inputs = query\n",
        "            labels = np.empty(0)\n",
        "\n",
        "        targets = labels + 1\n",
        "        inputs = pad_batch(inputs, self.seq_len, self.pad_value)\n",
        "        targets = pad_batch(targets, self.seq_len, self.pad_value)\n",
        "\n",
        "        if self.is_local:\n",
        "            labels = np.r_[labels, query_label]\n",
        "            labels = pad_batch(labels, self.seq_len,\n",
        "                               self.pad_value)\n",
        "            uids = np.full(inputs.shape[0], uid)\n",
        "            inputs = np.c_[inputs, uids, labels]\n",
        "\n",
        "        return np.c_[inputs[1:], targets[:-1]].astype(np.float32)\n",
        "\n",
        "\n",
        "def collate_fn_test(batch):\n",
        "    return np.array(batch).transpose(2, 0, 1)\n",
        "\n",
        "\n",
        "def update_stats(prev_data, prev_batch):\n",
        "    dcols = {col: i for i, col in\n",
        "             enumerate([KEY_FEATURE] + FEATURES + \n",
        "                       [TARGET, 'timestamp_raw'])}\n",
        "\n",
        "    def update_stat(trow):\n",
        "        uid = trow[0]\n",
        "        if uid in prev_data:\n",
        "            last_timestamp = prev_data[uid][-1, dcols['timestamp_raw']]\n",
        "            curr_timestamp = trow[dcols['timestamp']]\n",
        "            diff_timestamp = np.ceil((curr_timestamp - last_timestamp) / \n",
        "                                     TIME_SCALES['timestamp'])\n",
        "            trow[dcols['timestamp']] = diff_timestamp\n",
        "            trow = np.r_[trow, curr_timestamp]  # Add timestamp_raw of the query\n",
        "            prev_data[uid] = np.r_[prev_data[uid][-SEQ_LEN + 2:], [trow]] \\\n",
        "                .astype(int)\n",
        "        else:\n",
        "            curr_timestamp = trow[dcols['timestamp']]\n",
        "            trow[dcols['timestamp']] = DEFAULT_VALUES['timestamp']\n",
        "            trow = np.r_[trow, curr_timestamp]  # Add timestamp_raw of the query\n",
        "            prev_data[uid] = np.array([trow])\n",
        "            \n",
        "    np.apply_along_axis(update_stat, arr=prev_batch, axis=1)\n",
        "\n",
        "\n",
        "def predict_test(estimator,\n",
        "                 tst_iterator,\n",
        "                 local=False,\n",
        "                 prev_data=None,\n",
        "                 device=\"cpu\"):\n",
        "    target_idx = -1\n",
        "    estimator.eval()\n",
        "\n",
        "    outs = np.empty(0)\n",
        "    for batch in tst_iterator:\n",
        "        inputs = {}\n",
        "        for i, feat in enumerate(FEATURES):\n",
        "            if DTYPES[feat] is int:\n",
        "                inputs[feat] = torch.Tensor(batch[i].astype(int)) \\\n",
        "                    .to(device).long()\n",
        "            elif DTYPES[feat] is float:\n",
        "                inputs[feat] = torch.Tensor(batch[i]).to(device).float()\n",
        "            elif DTYPES[feat] is bool:\n",
        "                inputs[feat] = torch.Tensor(batch[i].astype(bool)) \\\n",
        "                    .to(device).bool()\n",
        "        inputs['targets'] = torch.Tensor(batch[target_idx].astype(np.int64)) \\\n",
        "            .to(device).long()\n",
        "\n",
        "        with torch.no_grad():\n",
        "            output = estimator(inputs=inputs)[:, -1]\n",
        "        outs = np.r_[outs, output.data.cpu().numpy()]\n",
        "\n",
        "    return outs\n",
        "\n",
        "\n",
        "def predict_submission_group(estimator,\n",
        "                             tst_batch,\n",
        "                             prev_batch,\n",
        "                             prev_data,\n",
        "                             batch_size=128):\n",
        "    all_cols = list(tst_batch.columns) + ADDED_FEATURES + [TARGET]\n",
        "    all_cols = dict(zip(all_cols, range(len(all_cols))))\n",
        "    used_cols = [all_cols[feat] for feat in [KEY_FEATURE] + FEATURES]\n",
        "    tst_batch = tst_batch.values\n",
        "\n",
        "    if (prev_batch is not None) & (psutil.virtual_memory().percent < 95):\n",
        "        prev_batch = np.c_[prev_batch, eval(\n",
        "            tst_batch[0, all_cols['prior_group_answers_correct']])]\n",
        "        prev_batch = prev_batch[prev_batch[:, all_cols['content_type_id']] == 0\n",
        "                                ][:, used_cols + [all_cols[TARGET]]]\n",
        "        update_stats(prev_data, prev_batch)\n",
        "\n",
        "    default_values = [DEFAULT_VALUES[feat] for feat in ADDED_FEATURES]\n",
        "    question_feats = np.apply_along_axis(\n",
        "        lambda rid: [QUESTIONS_DF[rid[0]][feat] for feat in ADDED_FEATURES]\n",
        "        if rid[0] in QUESTIONS_DF else default_values,\n",
        "        axis=1, arr=tst_batch[:, [all_cols['content_id']]])\n",
        "    tst_batch = np.c_[tst_batch, question_feats]\n",
        "    tst_batch = preprocess_test(tst_batch, all_cols)\n",
        "    prev_batch = tst_batch.copy()\n",
        "\n",
        "    qrows = tst_batch[:, all_cols['content_type_id']] == 0\n",
        "    tst_batch = tst_batch[qrows]\n",
        "    tst_dataset = RiiidTest(prev_data, tst_batch[:, used_cols], SEQ_LEN)\n",
        "    tst_dataloader = torch.utils.data.DataLoader(dataset=tst_dataset,\n",
        "                                                 batch_size=batch_size,\n",
        "                                                 collate_fn=collate_fn_test,\n",
        "                                                 num_workers=0)\n",
        "\n",
        "    _preds = predict_test(estimator, tst_dataloader, device=DEVICE)\n",
        "    tst_batch = np.c_[tst_batch, _preds]\n",
        "    _predictions = pd.DataFrame(\n",
        "        tst_batch[:, [all_cols[col] for col in SUBMISSION_COLUMNS]],\n",
        "        columns=SUBMISSION_COLUMNS)\n",
        "\n",
        "    return {'preds': _predictions,\n",
        "            'prev_batch': prev_batch,\n",
        "            'prev_data': prev_data}\n",
        "\n",
        "\n",
        "def load_previous_data(pdata_path):\n",
        "    pdata = preprocess(load_data(pdata_path, cols=[KEY_FEATURE] + \n",
        "                                 FEATURES +  [TARGET, 'timestamp_raw']))\n",
        "    gc.collect()\n",
        "    print(f'getting {SEQ_LEN} long sequences...')\n",
        "    pdata = pdata.groupby(KEY_FEATURE).tail(SEQ_LEN)\n",
        "    pdata = pdata.groupby(KEY_FEATURE).apply(lambda g: g.values)\n",
        "    gc.collect()\n",
        "    qdata = pd.read_parquet(PARQUETS_DIR + \n",
        "                            'questions.parquet').to_dict('index')\n",
        "    return pdata, qdata"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yqTFRyD0Fup6"
      },
      "source": [
        "# Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hTo73W_XFk5f"
      },
      "source": [
        "MODEL = TransformerModel  # @param [\"TransformerModel\"] {type:\"raw\"}\r\n",
        "\r\n",
        "MODEL_FILENAME = f'{MODEL.__name__}_best.pth'  # @param {type:\"string\"}\r\n",
        "\r\n",
        "\r\n",
        "# @markdown # Model Settings\r\n",
        "\r\n",
        "SEQ_LEN = 257  # @param {type:\"integer\"}\r\n",
        "\r\n",
        "D_MODEL = 256  # @param {type:\"integer\"}\r\n",
        "\r\n",
        "NHEAD = 8  # @param {type:\"integer\"}\r\n",
        "\r\n",
        "N_ENC_LAYERS = 2  # @param {type:\"integer\"}\r\n",
        "\r\n",
        "N_DEC_LAYERS = 2  # @param {type:\"integer\"}\r\n",
        "\r\n",
        "DIM_FEEDFORWARD = 2048  # @param {type:\"integer\"}\r\n",
        "\r\n",
        "DROPOUT = 0.1  # @param {type:\"slider\", min:0, max:1, step:0.05}\r\n",
        "\r\n",
        "ACTIVATION = \"gelu\"  # @param [\"relu\", \"gelu\"]\r\n",
        "\r\n",
        "LEARNING_RATE = 0.0001  # @param {type:\"number\"}\r\n",
        "\r\n",
        "\r\n",
        "# @markdown # Training Settings\r\n",
        "\r\n",
        "EPOCHS = 100  # @param {type:\"integer\"}\r\n",
        "\r\n",
        "BATCH_SIZE = 64  # @param {type:\"integer\"}\r\n",
        "\r\n",
        "EARLY_STOPPING = 10  # @param {type:\"integer\"}\r\n",
        "\r\n",
        "NUM_WORKERS = 0  # @param {type:\"integer\"}\r\n",
        "\r\n",
        "SCHED_TYPE = \"plateau\"  # @param [\"plateau\", \"lambda\", \"NONE\"]\r\n",
        "\r\n",
        "WARMUP_STEPS = 4000  # @param {type:\"integer\"}\r\n",
        "\r\n",
        "\r\n",
        "\r\n",
        "\r\n",
        "PAD_VALUE = 0\r\n",
        "DEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\") \\\r\n",
        "    if not TPU else xm.xla_device()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mUGqhFm4BlcE"
      },
      "source": [
        "RETRAIN = True  # @param {type:\"boolean\"}\r\n",
        "\r\n",
        "CONTINUE = False  # @param {type:\"boolean\"}\r\n",
        "\r\n",
        "DEBUG = False  # @param {type:\"boolean\"}\r\n",
        "\r\n",
        "DATA_SIZE = 1  # @param {type:\"slider\", min:0.01, max:1, step:0.01}\r\n",
        "\r\n",
        "DEBUG_SIZE = 0.1  # @param {type:\"slider\", min:0.01, max:1, step:0.01}\r\n",
        "\r\n",
        "LOCAL_SAMPLE = False  # @param {type:\"boolean\"}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mt9ZKyhyDFyC",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "df84cc7c15cb4a1b88e70a68aa036818",
            "8cd2a5459b8d4ec2bf4bc8c1edbdf3ab",
            "65aab87ad9d7436cbc0572958bdc9e74",
            "a35b7017d4264b7cbe6fab067cfe2c59",
            "9e782383c65f417faecb62a8190a8e2d",
            "726b7c3dd6d047e9b9af04c996575e95",
            "d434ff4399224932804a0b26d2b076e8",
            "8f97a2ea92d34de28eba6bae186b809f",
            "473ffb66bc0d416d916f21a5570f70dc",
            "63e623ca8729418580a2fbec217269de",
            "7c0c9fd2c6254d028b6aa2d244d8a933",
            "3bd813074b0a4fc596c67358b8b6562d",
            "f2e3b278ea1b423ebcedb24aef7410d0",
            "32955a66bec74c319e8c29a366ba7974",
            "e0c9ac4b120948dea71dce16902855fa",
            "6b9c009ff067445ea1b8b8db94deb385",
            "5633ea47bf8949c98c566c20e11619f1",
            "a86258d956b847aca01a1eea2d3c6140",
            "b80f2471c42047b79b6d4e1317fd1544",
            "28fd6161cfa44cc3b8cb98d715b91567",
            "cc1d542440c645c4b4d705fdd2afdfa4",
            "d4567437f0b145b7832dd07907640492",
            "6d7913187b2647e9a60a6de87ca55b19",
            "18e756f6f1a14287ae4c79b96a1b887f",
            "64dfa90f814642a6bc2a214a940aa52f",
            "ed70d94f4ef842d1b31269789d2308d6",
            "748c2259deef467daf5efaeba3bc5e76",
            "81423ca9914244809e076744a9819db1",
            "81a85f534c41473c896bc0baad5a02cc",
            "4814cf50c5c54e28afe17d5d08d9abdd",
            "39797611ecea492b9b80a8a68a1dbe10",
            "76a403dee600497eb77c176a1a67b9a5",
            "b8d47ead000d4c468872e1f8d11b67b1",
            "a8dfaa3c6f564b4a94c4b1f4922076dc",
            "8629e2e1e93049539cf7d6b33e2a78e9",
            "4ffb0ccea2104ccca7858b5eb833e75a",
            "351b52d5b5df412f9dd5ec45bebee6fb",
            "0daeac05956d45faaa6c862f43adca75",
            "4074cb065e074d6da0ffda6346755389",
            "d1aa2522d2634e1486d746efcbb89d46",
            "d97a665a2d4942c0b286b58f9dad980a",
            "0a6cba75aed64e869d4919acee9baaff",
            "f1391614523f4d73917ff02218c17d6d",
            "7c38f83c31bf4172866eb33e5d9fdff0",
            "954cdda3f30b417294bec65bea33438b",
            "5f461a66e5e14ba8bf1cfb7100ac9f6f",
            "8950813b9b0f47d48a53c4202eb796f2",
            "11aef95f54c0414e89ea39c8b3aa3a23",
            "7e08544db5b24a0591b2c29f2f83cbc2",
            "2057839f46754054beb005d9499d3fea",
            "d115acbecbb8462e9d68df1f35a7c95a",
            "cd88ff31d79e4335a987c1c2be2e25cd",
            "e1fc7cf4e61c4553b9ca80a23c9f4ba7",
            "6c1d9653a12a4e54924e886bc5110511",
            "30d6f4d93ca141358ca0e013ef6cae3b",
            "787c7cb1659e4f34a0dd4928237f924d",
            "93807d8733a145db9193b9241d8b523f",
            "958dfebe4566473a9ccdd5c321958606",
            "fedad49e61eb456c94ed3971972add64",
            "18b876b51c4e45078b1f8e58a0d98ec4",
            "dc397bc492fd484f993a3cc7e334065f",
            "e9e220fb53b045a18eef15c7c946450c",
            "e68519bf458d4116b3ea62249fe95bf4",
            "5b2ca36c9a90486eae8b3d0fb35e2d03",
            "9dbb73cd94fd4127bde382815b1af77f",
            "3b8b2e0e62e04833be9981384a4b4a96",
            "ad1bfa22b56942af97513314b1757a7c",
            "a8066afadbdf49f5b006210507cef03f",
            "08309fbf48614fbab5cd3a8b93796298",
            "7e0b04cd460a42228a72dc20b8cbad71",
            "d202fc6fa9d24f828a2e98a744493f55",
            "f53e1cb3e2ba4fc6aff08600a135028f",
            "8bc8439567d84e8aa740da7da70aebc1",
            "a58533372826430c866887446caed324",
            "e0e2ab6e40ae4c20853aa0f937ef4e0f",
            "ac64d71f1ff2402284064dd6a1ddc334",
            "607d25118f054e3da33f4675884e831b",
            "877f23d85a7a4f618854932a980a25ed",
            "58a4820ff6384e9b8386790c97608430",
            "6fef47633b9347eebd1f3d2b4a86c578",
            "ab47eb143dcf4a22b2b5e5767a0c6f17",
            "fc9a7acd627e4f3b8479ba9c51c036e3",
            "ecfcfff08f6946ad99c8d31d522c4f28",
            "0ea33d9760ba4902ad001ab40c09d889",
            "cc81de2270f3422bbfe6636671820d9f",
            "32325be2b3774d3e800a965501a0a8dc",
            "0e85cb960e0c4606aea3e34ab7055982",
            "3c9f62d01b73467d8ab9e7ae0b132d88",
            "9ec375f237a240e781998269b6395abc",
            "1350bee07c0b4b0a9e01bba9c120bb45",
            "fc0e40812257448f89288dc6ce5d0f02",
            "0973eb052a1444ae90ef01b713d7e417",
            "ca3e3706229045ec8a222948550d9aa2",
            "74493df8dd55443fa0377ffea6c53bed",
            "a81111a74a06434e8a1266b01b0f25c9",
            "4e1b66fa1f7b4ccbaa42e23e9d36b2d3",
            "18ef3d0dd4ff48c2a811e3457052f982",
            "8b89751502134a719a01ca5f8d9bcf03",
            "85297db351b444ba956b85295a985974",
            "1da526007a254f22977ebd3dd293d359",
            "922f8d69728f4fd3bb7a78e4cfe744fe",
            "03f9313a50ae43b293f1ca325aa281b1",
            "13a60bc1bdfb484ba5156323c547bae5",
            "49b26633b4e14512a54848c4eeed53da",
            "ccaaecf15ebc424aa14465f8b824bfcb",
            "b0a1dbea033d4fc3b8c8ef9756d4340e",
            "4b632a37d4c44e88ae9cd3cecc06704d",
            "8d8667b0e885487c9a19d4af76b0e3a4",
            "25c241b7fc764bcfb2a0c2f8805652b4",
            "b3a3d6b94c7b49c984e17207b1b443d4",
            "b05db9b1b82a49b68b468d86f83edefe",
            "c20fdd5e18e849658105db5a6f2c4d35",
            "6519bbce4ab44f66957783b18f2b620a",
            "b8f514d2a6d4432790f57cfecd177b27",
            "e2eec45395fa4ee592cee7d44d31ddd2",
            "93011fdd31da4960bb2b4a23c6fe16aa",
            "3259289a58744f00b7812bd828d6c80a",
            "cfe767fb0c284e5e85a20d76ed889816",
            "f13d5ccc16664e91955fa6feaa1eb47b",
            "797d355953f84092a5ff58030473ce6c",
            "2c618ff1773a48e7a90b5a72276ed61a",
            "5bd58ccd9ef94846ab55fd5016de4484",
            "a16ef1024a6746aab9765fb857a3393d",
            "1e66d54bb5974786b1e748a449d3ef82",
            "bbc5292245334a7ab6c9980530ca0574",
            "42425a08a94d4aa19dcc50bf5f2d40d3",
            "5c7e7e97f77c4b3abe62fcc6084e1fe4",
            "a82b12d43c5f4f788b05372c8284bba0",
            "b57c885e5527452c93bdce02ed271736",
            "1fe7e69d181e40adb49893ebc383bc21",
            "8eab1b941a4a4aeeba4527fc32ee4cf8",
            "eae4cc7618714ad6b60c7e1fa204a3c1",
            "41dde5caf17d43e198ffab849856c801",
            "7e8eead0049e44079d0c262b74f9b677",
            "94b6048cad064ff3a173b193f70675d0",
            "f058c3dafa73478d8266d71c61cef939",
            "efd169582d1a4ef3aebcd6ce7d4a1c56",
            "7fa3a75698dd46619445845995638d29",
            "e41e1fc358cf488bbc6d799d4e3f45f2",
            "27595a5587e64b23b621fe7c7375db9f",
            "1fd31b4755ef48a2be529a0bcdf30865",
            "613f122146ce402b89bc0e793f50c7c3",
            "ae1514384d884fe99541a1556d32ee4e",
            "a96705dbaa2d48a098353f8c845aa63e",
            "d4c74d600f68455d9c16cf2f2b6572a7",
            "bc1e78b35e5b4edc9f7a831be9607cc5",
            "f0c78d186afa4c36875453acceaf841f",
            "5a8eb97172674d1db8245db3548e3616",
            "88852086665d46158d33c1ce4857b321",
            "6a85791520314394b55a61e0739e1bf9",
            "66a22d4a828b4517a973396a40ac6367",
            "7de635702d5440beb0851fd8eb0a4320",
            "d9f13eafbebf464e9e9f32f357dbfade",
            "79c139a4052a4d329203049244fb584c",
            "d09a67112d444435931d22c8368c6579",
            "b8e0cf5cb2464ba48517f78e6f294197",
            "b394d83a302c4ca5902d454b1ab9cfc0",
            "3c93b9eb3bad48e2a8d8c8389881e7b0",
            "b34b3e4282584d5d8cf22dfb4e7fc8bc",
            "2138f1ea38f54c6898220b3d20fda7b9",
            "1d065d84ec1e4c5ba5b566ecca396d5f",
            "ecbfd200a6e74105b8f1a9411d67e1d4",
            "77898901c5c04a2b857c5acfd56e23f3",
            "1f53eb2889b747cb80b7e61285ea2ea7",
            "fe93afa91f2443db8f20238b49250c78",
            "9532e6859bca4b3bb240477d1a386014",
            "9133ed1a74494cae8988af6a88a6da62",
            "05cfa69143f048b69133a28384a9e2ad",
            "1509ecd840454564aa2e904550d74e89",
            "67afd9a8ec274e36b6d73e1dee2af7af",
            "3b6331cd0bb84f179edc693827c28a0c",
            "3f2e3f858f6644c7bcf75e611b69647a",
            "712de39fc28b482481a9961940f523a4",
            "e488599981c04ba488c0551f382a19c4",
            "cf494adc90e24298b6cc650a12037fe8",
            "16494f4cdffa45b48ba7ae05f645ee8d",
            "a55ea250187b4f59b7929fb0727ba258",
            "2037e2dc30cc42479e152f570e89d132",
            "44824f2cd5cc42e48a4568ff7d54e92f",
            "582a7ecacc3b42078b10f1f682294474",
            "f465867c37a9459e90b272a58535f4f9",
            "904ba60891234c2fa0cec710920944b3",
            "1d9094decfd34c559d20233f88a6a8e8",
            "caf29b258f384d5c9eab60e1e779ab5a",
            "101464caa8484e2a9c4aca479a8e2a65",
            "acbdff72abc84fbaafbbe1eef4130c06",
            "76f50cdfb551492cbcd984db172b4a76",
            "804b2180e3794ada8dd9681a4a917ab2",
            "73b80cc30a734f68a74d973d1b071bff",
            "6b5736e5c67441ad884d897dae8e6820",
            "84c034317f7e4a6782e8aa6c46cc4020",
            "d49cdf26c1f7447cbb1982970b2f97d5",
            "53971220d6cd4bba8fc56aea264a8dbc",
            "ba09fccf1b8d4b97b496d5eb9bd8c057",
            "1d7afe011cb04d448df48db56006fabe",
            "1161ce97641441f1b68cd53a18529005",
            "942a9b14b8cf44b99cc7411db3a34690",
            "69ec20cd5b584253a30984463f2092a7",
            "a26bb8f3ba03406c840e0726cc2a4a65",
            "63bd86e0244844eeba38ea8b7548b26d",
            "a0cbb986aa7d42389add69fab0e18937",
            "782fab00eb074740ab96d329735b4373",
            "dfd8c94343944dd3a6819f8b6340a1f1",
            "1ef3adef5efc47fd8c1be63d530f9127",
            "002dcae14957488ea1bba489bb86635c",
            "a56662f45e8148adb9e956684c99fdd0",
            "976ec2ccb3744aa1b6d0c68c06571391",
            "70aa75fe3df945efa8c851b1e6abe824",
            "ec7525f3906e4ca8850ecbace87e0dfe",
            "3270d641b02b4726933162253a80427e",
            "4c0360e1c149420fa6efe800ed3fb106",
            "1786bc1e9bea4e5b9d3d73d2a56aa4ee",
            "9b793107a8cd40f6b83679b1874100bb",
            "3df13214a39a4f62835fc4c6c6322d3c",
            "f44d73ca57184d819742673784bf3d19",
            "5715a2d871704d6cba7f8e45ca933fc7",
            "4e02369b94b548af9cb10c6473bfb616",
            "8b5b2b399ef147dbb0636f1fe1a537e9",
            "463de4c99cb1412c800da584c2fb63ca",
            "a78e20209f1a4f2d8971c623a943049e",
            "9fc1c0bbe2d54f79bafa50861c4da31f",
            "3ffe2bd239dc4b1d802af45a193d1f88",
            "13dd00d1fd774db9a6177710958d8e56",
            "fdfddd25e161418a9cac39e30ca36ab6",
            "12f1bc6fa1fc427abd216de8539da1f6",
            "3f86b65af0e94c18a7db4859f3ac4082",
            "76152c60c3fe4440a8497510b0305c1c",
            "a887db49709d48558d7381215dd7e2a0",
            "4df05383921b404a92f5c4d2b1b07a5a",
            "d213521d5e154771a2521ba715bf4544",
            "13c6ad736acd455ba471b24b197b66b4",
            "84e94fe45b7e4d429d64275de40268ef",
            "b836b3b015f14c2abd7f8b279a823ea9",
            "b0500040a7b84927b5b2360b175bf224",
            "c5a281682f0b4b20943b80d4e5ea743c",
            "a8a7cf017d57425281d0d5d90c3e8391",
            "1137ab97d3cd47288d92a6b50097ba4d",
            "8d1ce78aac9746fea32c1fdf5975d909",
            "11dc6f2c2fca4fb7a12342e9a1784348",
            "dbea74a8f70b44a19bdac2487e5f6e1f",
            "1f856acceb304e1ab4a7b5475ecf20c0",
            "26eae43436ec46e3931109b2c1e7f915",
            "7fd08036b71c46f5b2f4f466484a12c2",
            "bbb282986ab14d3eac6595be86ae3b17",
            "1c0f4f33b0c247d396587f7a89330ebc",
            "2a31c5d85aef4442a333826b9f009885",
            "02f3ce638a1742939b346fe1363d9aa3",
            "c61a818da11c4cfa9f4bec4eed814058",
            "fca05926b73b4e128166792ec2d6b1d4",
            "ea027488b83d41c48013d8d305974f29",
            "9f07715a942043b6a0bc1a319440ebf0",
            "891a5c0098d14fc28cc03f1e4de60d12",
            "2bb101a82e5a44b3a2934c79e6820060",
            "5fa8aed958664c98a94bb962ce59a1ee",
            "fa0b244b33bf4685a5c80705171e5eb1",
            "607ceab609db4fe2bda86bc4f083b575",
            "b8e99e932a9345bebb42c653c69e62f5",
            "0c1a1fa9c56740a28c1f337d92ebe902",
            "acb91b4772db46639b92bd7d67e54bc2",
            "f3a4c9b3d76a4fbeaaa6266f600766dc",
            "1ff4eb6c8d6c49cbbb0b658eeae5b96d",
            "80d1f472fe174717a009feb8b6e486c2",
            "9011e41760304c39be30187606e91db4",
            "8a78bcdb72a94ad4a39239db1a42f8e3"
          ]
        },
        "outputId": "8939a1c5-5fe0-4fcd-8836-bf1469d19aa2"
      },
      "source": [
        "if __name__ == '__main__':\r\n",
        "    print('Using Device -', DEVICE)\r\n",
        "    print('Using Model -', MODEL.__name__)\r\n",
        "    model_params = {\r\n",
        "        'seq_len': SEQ_LEN,\r\n",
        "        'd_model': D_MODEL,\r\n",
        "        'nhead': NHEAD,\r\n",
        "        'num_encoder_layers': N_ENC_LAYERS,\r\n",
        "        'num_decoder_layers': N_DEC_LAYERS,\r\n",
        "        'dim_feedforward': DIM_FEEDFORWARD,\r\n",
        "        'dropout': DROPOUT,\r\n",
        "        'activation': ACTIVATION\r\n",
        "    }\r\n",
        "\r\n",
        "    train_params = {\r\n",
        "        'estimator_type': MODEL,\r\n",
        "        'estimator_params': model_params,\r\n",
        "        'epochs': EPOCHS,\r\n",
        "        'batch_size': BATCH_SIZE,\r\n",
        "        'early_stopping': EARLY_STOPPING,\r\n",
        "        'nworkers': NUM_WORKERS,\r\n",
        "        'device': DEVICE,\r\n",
        "        'eps': 1e-6,\r\n",
        "        'cont': CONTINUE,\r\n",
        "        'debug': DEBUG,\r\n",
        "        'data_size': DATA_SIZE,\r\n",
        "        'debug_size': DEBUG_SIZE,\r\n",
        "        'valid_size': 0.1\r\n",
        "    }\r\n",
        "\r\n",
        "    if RETRAIN and not COMPILE:\r\n",
        "        print('Parameters:', train_params)\r\n",
        "        if TPU:\r\n",
        "            xmp.spawn(tpu_map_fn, args=(train_params,), nprocs=8, \r\n",
        "                      start_method='fork')\r\n",
        "        else:\r\n",
        "            model = train_transformer(**train_params)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using Device - cuda:0\n",
            "Using Model - TransformerModel\n",
            "Parameters: {'estimator_type': <class '__main__.TransformerModel'>, 'estimator_params': {'seq_len': 257, 'd_model': 256, 'nhead': 8, 'num_encoder_layers': 2, 'num_decoder_layers': 2, 'dim_feedforward': 2048, 'dropout': 0.1, 'activation': 'gelu'}, 'epochs': 100, 'batch_size': 64, 'early_stopping': 10, 'nworkers': 0, 'device': device(type='cuda', index=0), 'eps': 1e-06, 'cont': False, 'debug': False, 'data_size': 1, 'debug_size': 0.1, 'valid_size': 0.1}\n",
            "Using Columns - ['user_id', 'content_id', 'timestamp', 'prior_question_elapsed_time', 'prior_question_had_explanation', 'part', 'tag1', 'tag2', 'answered_correctly']\n",
            "total size: (99271300, 9)\n",
            "data size: (99271300, 9) - num users: 393656\n",
            "train size: (89344170, 9) - num users: 354295\n",
            "valid size: (9927130, 9) - num users: 39361\n",
            "{'seq_len': 257, 'd_model': 256, 'nhead': 8, 'num_encoder_layers': 2, 'num_decoder_layers': 2, 'dim_feedforward': 2048, 'dropout': 0.1, 'activation': 'gelu'}\n",
            "\n",
            "Epoch 1 - Learning rate: 0.0001\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "df84cc7c15cb4a1b88e70a68aa036818",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.561163 - acc: 0.709668\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "473ffb66bc0d416d916f21a5570f70dc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Validation -- loss: 0.541736 - acc: 0.724489 - auc: 0.761726 - custom_auc: 0.766612\u001b[0m\n",
            "\n",
            "Epoch 2 - Learning rate: 0.0001\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5633ea47bf8949c98c566c20e11619f1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.541063 - acc: 0.722509\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "64dfa90f814642a6bc2a214a940aa52f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\u001b[91m  Validation -- loss: 0.525650 - acc: 0.732457 - auc: 0.776894 - custom_auc: 0.776189\u001b[0m\n",
            "\n",
            "Epoch 3 - Learning rate: 0.0001\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b8d47ead000d4c468872e1f8d11b67b1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.533975 - acc: 0.726331\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d97a665a2d4942c0b286b58f9dad980a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\u001b[91m  Validation -- loss: 0.522515 - acc: 0.734564 - auc: 0.780570 - custom_auc: 0.780706\u001b[0m\n",
            "\n",
            "Epoch 4 - Learning rate: 0.0001\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7e08544db5b24a0591b2c29f2f83cbc2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.530640 - acc: 0.728371\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "93807d8733a145db9193b9241d8b523f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\u001b[91m  Validation -- loss: 0.520654 - acc: 0.735573 - auc: 0.782469 - custom_auc: 0.782450\u001b[0m\n",
            "\n",
            "Epoch 5 - Learning rate: 0.0001\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9dbb73cd94fd4127bde382815b1af77f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.528391 - acc: 0.729845\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8bc8439567d84e8aa740da7da70aebc1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\u001b[91m  Validation -- loss: 0.518911 - acc: 0.736614 - auc: 0.784251 - custom_auc: 0.784358\u001b[0m\n",
            "\n",
            "Epoch 6 - Learning rate: 0.0001\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ab47eb143dcf4a22b2b5e5767a0c6f17",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.526218 - acc: 0.731366\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9ec375f237a240e781998269b6395abc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\u001b[91m  Validation -- loss: 0.517989 - acc: 0.737284 - auc: 0.785477 - custom_auc: 0.785865\u001b[0m\n",
            "\n",
            "Epoch 7 - Learning rate: 0.0001\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "18ef3d0dd4ff48c2a811e3457052f982",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.524382 - acc: 0.732511\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ccaaecf15ebc424aa14465f8b824bfcb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\u001b[91m  Validation -- loss: 0.517307 - acc: 0.738070 - auc: 0.786610 - custom_auc: 0.786990\u001b[0m\n",
            "\n",
            "Epoch 8 - Learning rate: 0.0001\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6519bbce4ab44f66957783b18f2b620a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.522850 - acc: 0.733647\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2c618ff1773a48e7a90b5a72276ed61a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\u001b[91m  Validation -- loss: 0.516803 - acc: 0.738763 - auc: 0.787397 - custom_auc: 0.787486\u001b[0m\n",
            "\n",
            "Epoch 9 - Learning rate: 0.0001\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b57c885e5527452c93bdce02ed271736",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.521206 - acc: 0.734798\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "efd169582d1a4ef3aebcd6ce7d4a1c56",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\u001b[91m  Validation -- loss: 0.515382 - acc: 0.739443 - auc: 0.788345 - custom_auc: 0.789655\u001b[0m\n",
            "\n",
            "Epoch 10 - Learning rate: 0.0001\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d4c74d600f68455d9c16cf2f2b6572a7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.519619 - acc: 0.735936\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d9f13eafbebf464e9e9f32f357dbfade",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Validation -- loss: 0.514885 - acc: 0.739585 - auc: 0.788606 - custom_auc: 0.788490\u001b[0m\n",
            "\n",
            "Epoch 11 - Learning rate: 0.0001\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1d065d84ec1e4c5ba5b566ecca396d5f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.518072 - acc: 0.737111\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1509ecd840454564aa2e904550d74e89",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Validation -- loss: 0.514851 - acc: 0.739956 - auc: 0.788941 - custom_auc: 0.789917\u001b[0m\n",
            "\n",
            "Epoch 12 - Learning rate: 0.0001\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a55ea250187b4f59b7929fb0727ba258",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.516610 - acc: 0.738038\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "101464caa8484e2a9c4aca479a8e2a65",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Validation -- loss: 0.514795 - acc: 0.740146 - auc: 0.789243 - custom_auc: 0.790131\u001b[0m\n",
            "\n",
            "Epoch 13 - Learning rate: 0.0001\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "53971220d6cd4bba8fc56aea264a8dbc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.514995 - acc: 0.739217\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a0cbb986aa7d42389add69fab0e18937",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Validation -- loss: 0.514954 - acc: 0.740422 - auc: 0.789468 - custom_auc: 0.791141\u001b[0m\n",
            "\n",
            "Epoch 14 - Learning rate: 0.0001\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ec7525f3906e4ca8850ecbace87e0dfe",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.513208 - acc: 0.740425\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4e02369b94b548af9cb10c6473bfb616",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Validation -- loss: 0.515881 - acc: 0.740163 - auc: 0.789144 - custom_auc: 0.790713\u001b[0m\n",
            "\n",
            "Epoch 15 - Learning rate: 5e-05\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "12f1bc6fa1fc427abd216de8539da1f6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.510193 - acc: 0.742446\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b836b3b015f14c2abd7f8b279a823ea9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Validation -- loss: 0.514358 - acc: 0.740656 - auc: 0.789868 - custom_auc: 0.792054\u001b[0m\n",
            "\n",
            "Epoch 16 - Learning rate: 5e-05\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1f856acceb304e1ab4a7b5475ecf20c0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Training -- loss: 0.508635 - acc: 0.743480\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fca05926b73b4e128166792ec2d6b1d4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=951.0), HTML(value='')), layout=Layout(di…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Validation -- loss: 0.516144 - acc: 0.740264 - auc: 0.789257 - custom_auc: 0.791602\u001b[0m\n",
            "\n",
            "Epoch 17 - Learning rate: 2.5e-05\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b8e99e932a9345bebb42c653c69e62f5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=6642.0), HTML(value='')), layout=Layout(d…"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OSp9PFh5u4v2"
      },
      "source": [
        "# Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FBdPuB3EIGKe"
      },
      "source": [
        "# ============================= TESTING ===================================\r\n",
        "print('TESTING')\r\n",
        "print('loading previous data...')\r\n",
        "previous_data, QUESTIONS_DF = load_previous_data('train_merged.parquet')\r\n",
        "model, _, _, _ = load_model(MODEL)\r\n",
        "model.eval()\r\n",
        "gc.collect()\r\n",
        "\r\n",
        "if not COMPILE:\r\n",
        "    print('Submitting locally...')\r\n",
        "    previous_batch = None\r\n",
        "    tgts = []\r\n",
        "    example_test = pd.read_csv(DATA_DIR + 'example_test.csv')\r\n",
        "    submission = pd.DataFrame(columns=SUBMISSION_COLUMNS)\r\n",
        "    for gnum in tqdm(example_test['group_num'].unique()):\r\n",
        "        test_batch = example_test[\r\n",
        "            example_test['group_num'] == gnum].copy()\r\n",
        "        preds, previous_batch, previous_data = predict_submission_group(\r\n",
        "            model, test_batch,\r\n",
        "            previous_batch,\r\n",
        "            previous_data,\r\n",
        "            batch_size=1024).values()\r\n",
        "        tgts.extend(\r\n",
        "            eval(test_batch['prior_group_answers_correct'].iloc[0]))\r\n",
        "        submission = submission.append(preds)\r\n",
        "    tgts.extend([-1] * len(test_batch))\r\n",
        "    submission['target'] = tgts\r\n",
        "    submission['pred'] = (submission[TARGET] >= 0.5).astype(np.int8)\r\n",
        "    submission = submission.reset_index(drop=True)\r\n",
        "    acc = submission[submission['target'] != -1]['target'] == \\\r\n",
        "          submission[submission['target'] != -1]['pred']\r\n",
        "    acc = sum(acc) / len(acc)\r\n",
        "    print(submission)\r\n",
        "    print('Accuracy:', acc)\r\n",
        "    print('AUC:', roc_auc_score(submission[submission['target'] != -1]['target'], \r\n",
        "                                submission[submission['target'] != -1]['pred']))\r\n",
        "else:\r\n",
        "    print('Submitting...')\r\n",
        "    env = riiideducation.make_env()\r\n",
        "    previous_batch = None\r\n",
        "    for test_batch, _ in env.iter_test():\r\n",
        "        preds, previous_batch, previous_data = \\\r\n",
        "            predict_submission_group(model,\r\n",
        "                                     test_batch,\r\n",
        "                                     previous_batch,\r\n",
        "                                     previous_data,\r\n",
        "                                     batch_size=1024).values()\r\n",
        "        env.predict(preds)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}