From 50ca7d17df1b57f07518a773c03d81f3ffef1458 Mon Sep 17 00:00:00 2001 From: Praxal Patel Date: Sat, 16 May 2020 14:30:00 -0400 Subject: [PATCH] Updating the Implementation code --- .DS_Store | Bin 0 -> 6148 bytes Bert_score_calculation.ipynb | 5112 ++++++++++++++++++++++++++ ROUGE_score_calculator.ipynb | 245 ++ Sentence_Bert_Score_calculator.ipynb | 267 ++ Sentence_Embedding.ipynb | 1148 ++++++ XLNet_Embedding.ipynb | 1 - 6 files changed, 6772 insertions(+), 1 deletion(-) create mode 100644 .DS_Store create mode 100644 Bert_score_calculation.ipynb create mode 100644 ROUGE_score_calculator.ipynb create mode 100644 Sentence_Bert_Score_calculator.ipynb create mode 100644 Sentence_Embedding.ipynb delete mode 100644 XLNet_Embedding.ipynb diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..f3a1c4e9b479c4e1c98b8b7860ded761aedb71e7 GIT binary patch literal 6148 zcmeHKu}T9$5Pcg{L~K%~u-HPuO0X8>IC6-UV6cw4OOU{6BqoL3eY9X1-^?r_lBBQ^ z2{U2mEweK_dvA}M41ka?$5%iXK!-!HbxgHGWL?IVw36kB7<7*UL%gHH4D&5-H~dEi zHR2OssF|P z{$HJ3IRnnXVKNZf{XswHv!&X)^^{U;BgYemi1;;z>kv+~6w_BqagRfV_9zo#mas9T Qh2nn%lm=JMz@IYk4M#vv-2eap literal 0 HcmV?d00001 diff --git a/Bert_score_calculation.ipynb b/Bert_score_calculation.ipynb new file mode 100644 index 0000000..e45f18a --- /dev/null +++ b/Bert_score_calculation.ipynb @@ -0,0 +1,5112 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "Bert_score_calculation.ipynb", + "provenance": [], + "collapsed_sections": [], + "machine_shape": "hm" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.7" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "accelerator": "GPU", + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "ea594def47344a209271cb3896a2829f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_e681a632b25f42cd87a32aa278078743", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_b2a334d1247148b68f0f5d1ffddab2da", + "IPY_MODEL_b0b91e181fa84d7194181b2d41093032" + ] + } + }, + "e681a632b25f42cd87a32aa278078743": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "b2a334d1247148b68f0f5d1ffddab2da": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_14815c9ba62b42a1b2b9aa5c232899e3", + "_dom_classes": [], + "description": "Downloading: 100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 434, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 434, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_6b6ef2bab7c8479a8cf38b863f079829" + } + }, + "b0b91e181fa84d7194181b2d41093032": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_9edeb2eb506f449d9dcf386bd3a95ac0", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 434/434 [00:02<00:00, 201B/s]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_91b3c31e94474d31a5ecc0568093fc6d" + } + }, + "14815c9ba62b42a1b2b9aa5c232899e3": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "6b6ef2bab7c8479a8cf38b863f079829": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "9edeb2eb506f449d9dcf386bd3a95ac0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "91b3c31e94474d31a5ecc0568093fc6d": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "ad048d2118aa4385800670defd5143b9": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_469b127ab1db46cca8dd856c3fcdfc9c", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_0e7cab84cb3541dba8d5f31de3ab3cb2", + "IPY_MODEL_f718b658101147c999f9824e03fbaf61" + ] + } + }, + "469b127ab1db46cca8dd856c3fcdfc9c": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "0e7cab84cb3541dba8d5f31de3ab3cb2": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_6cc8f68307cb462baec3d8118b304fb3", + "_dom_classes": [], + "description": "Downloading: 100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 231508, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 231508, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_39131ac8dff1423b9e1c2dcc1d1bc5bc" + } + }, + "f718b658101147c999f9824e03fbaf61": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_eec1c05633e7493bbbcbccac958aae52", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 232k/232k [00:00<00:00, 394kB/s]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_21451cf863b3413bbc80e22f1377ad86" + } + }, + "6cc8f68307cb462baec3d8118b304fb3": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "39131ac8dff1423b9e1c2dcc1d1bc5bc": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "eec1c05633e7493bbbcbccac958aae52": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "21451cf863b3413bbc80e22f1377ad86": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "05d3f607ffc14acbb44853ec5962bacc": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_31aa05a6211d486d89c162119f6ca064", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_4a8656f24cc345299f23a958c66488fd", + "IPY_MODEL_ef4dc6cfc73b4ee6a9636da04d6f0885" + ] + } + }, + "31aa05a6211d486d89c162119f6ca064": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "4a8656f24cc345299f23a958c66488fd": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_27fa5fd006a34745a017364859e5ca68", + "_dom_classes": [], + "description": "Downloading: 100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 1344997306, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 1344997306, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_6b2189d4ed8e47bd80aebbcacb9eec62" + } + }, + "ef4dc6cfc73b4ee6a9636da04d6f0885": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_6450eb81348b4ec4885f65c3bd0a0677", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 1.34G/1.34G [00:16<00:00, 81.9MB/s]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_7451929a3c574b368ddb6b39333d9513" + } + }, + "27fa5fd006a34745a017364859e5ca68": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "6b2189d4ed8e47bd80aebbcacb9eec62": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "6450eb81348b4ec4885f65c3bd0a0677": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "7451929a3c574b368ddb6b39333d9513": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "b170ede557524a73a32622fbe69818d6": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_4a45797271fe4e488441215249050c12", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_d49f5e320d5743da96526d5201d2cbf2", + "IPY_MODEL_19757705fb034fa796afe99829dbf97c" + ] + } + }, + "4a45797271fe4e488441215249050c12": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "d49f5e320d5743da96526d5201d2cbf2": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_6a7d84619c144f80913e10fa300f51ca", + "_dom_classes": [], + "description": "100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 354, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 354, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_edfdcc7322d4482c88e3c505ad7f8817" + } + }, + "19757705fb034fa796afe99829dbf97c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_07a8da43877c47e095d1727d233b70d0", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 354/354 [14:40<00:00, 2.49s/it]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_3ceffa85777a4102a87a1a411c44ea41" + } + }, + "6a7d84619c144f80913e10fa300f51ca": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "edfdcc7322d4482c88e3c505ad7f8817": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "07a8da43877c47e095d1727d233b70d0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "3ceffa85777a4102a87a1a411c44ea41": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "50ecdf9b42844436b79e6ae8b4cf42e1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_11ba2b7a69f148c8ba525a7b2f1a6306", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_e982c5477cd2437d81cdfa3ee45267dd", + "IPY_MODEL_d3d16ce0315943729ce8cc31eff046b7" + ] + } + }, + "11ba2b7a69f148c8ba525a7b2f1a6306": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "e982c5477cd2437d81cdfa3ee45267dd": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_08f4ad3701aa43d9a1659e7f32173451", + "_dom_classes": [], + "description": "100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 178, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 178, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_78fc3c361e3548559b9b8974bb3a5547" + } + }, + "d3d16ce0315943729ce8cc31eff046b7": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_4cf5a5aa86e04f8da0955fc894095ada", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 178/178 [00:39<00:00, 4.52it/s]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_1f4a0c088c2a49a2b10e28f88e67e35a" + } + }, + "08f4ad3701aa43d9a1659e7f32173451": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "78fc3c361e3548559b9b8974bb3a5547": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "4cf5a5aa86e04f8da0955fc894095ada": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "1f4a0c088c2a49a2b10e28f88e67e35a": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "3bf390f98f6e4782ab0779b87e99f8b1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_3f2215159b68490ab50ebc97ec4bfc66", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_f241740a91a946118703b160c16ed74a", + "IPY_MODEL_157a83855fca4d0a9c4b2a9c9774a56c" + ] + } + }, + "3f2215159b68490ab50ebc97ec4bfc66": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "f241740a91a946118703b160c16ed74a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_e2ac950e7f4649fca8ebd004b4d385e5", + "_dom_classes": [], + "description": "100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 353, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 353, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_4e5c6954853a44c7a4697341f4c6a46d" + } + }, + "157a83855fca4d0a9c4b2a9c9774a56c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_d2f616ae80334e8cbb0a18d48d273df3", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 353/353 [23:22<00:00, 3.97s/it]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_016afc9d449b434da2b49997d6951f3e" + } + }, + "e2ac950e7f4649fca8ebd004b4d385e5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "4e5c6954853a44c7a4697341f4c6a46d": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "d2f616ae80334e8cbb0a18d48d273df3": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "016afc9d449b434da2b49997d6951f3e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "ebccc2638e1d4dd1a233cb86bc3ca08e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_0f7098cf1dba49f8b6c2a65d20a57dc5", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_4c4564f0a6a74d76a9908e78b92be29f", + "IPY_MODEL_2fe9f75b2ec44152ac4e1f0801b7e503" + ] + } + }, + "0f7098cf1dba49f8b6c2a65d20a57dc5": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "4c4564f0a6a74d76a9908e78b92be29f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_0d4d41f947bb4c85a773154e93390fe1", + "_dom_classes": [], + "description": "100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 178, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 178, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_dbe54420e1074776892915293ead7724" + } + }, + "2fe9f75b2ec44152ac4e1f0801b7e503": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_43276d04629d484e8c1fb6b6318e894f", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 178/178 [01:29<00:00, 2.00it/s]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_45d8217658d64e2bb842dda215f8ed1b" + } + }, + "0d4d41f947bb4c85a773154e93390fe1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "dbe54420e1074776892915293ead7724": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "43276d04629d484e8c1fb6b6318e894f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "45d8217658d64e2bb842dda215f8ed1b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "ed55b90887f64449b12975722c797731": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_fa2b5c36f47a439998f38d4ce3f507d0", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_d866411994904eb58a0b275641410334", + "IPY_MODEL_8736181c78d9418e9cbf6bbd049e17b0" + ] + } + }, + "fa2b5c36f47a439998f38d4ce3f507d0": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "d866411994904eb58a0b275641410334": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_ad8929a3d9104769a26892f0e8eb5ea5", + "_dom_classes": [], + "description": "100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 354, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 354, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_a0605ad0cbdf4390a1f06320b53243c2" + } + }, + "8736181c78d9418e9cbf6bbd049e17b0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_730b04fe7078495d9e938fcca577f116", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 354/354 [13:40<00:00, 2.32s/it]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_cf86f43c2a004588adaafe0ff297d229" + } + }, + "ad8929a3d9104769a26892f0e8eb5ea5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "a0605ad0cbdf4390a1f06320b53243c2": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "730b04fe7078495d9e938fcca577f116": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "cf86f43c2a004588adaafe0ff297d229": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "d91a7532623b46d497e5edd861025711": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_be94e5d8259743ce91dd7455e2d11cb0", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_b124ae1458374db28aa89336119ee7cc", + "IPY_MODEL_32df47730e974a5eb56c40a948c83596" + ] + } + }, + "be94e5d8259743ce91dd7455e2d11cb0": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "b124ae1458374db28aa89336119ee7cc": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_80507eb3237f4218a2a7b98ccd617fbf", + "_dom_classes": [], + "description": "100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 178, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 178, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_9505637ab9f74f3d83ba71ae1683f1f0" + } + }, + "32df47730e974a5eb56c40a948c83596": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_7e3f6f793b2b4f0a9da89063e1344329", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 178/178 [04:46<00:00, 1.61s/it]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_09248031d64b4612932e51f237a4fbb2" + } + }, + "80507eb3237f4218a2a7b98ccd617fbf": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "9505637ab9f74f3d83ba71ae1683f1f0": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "7e3f6f793b2b4f0a9da89063e1344329": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "09248031d64b4612932e51f237a4fbb2": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "2ce06606f4fb4b7484a8531314f2fe03": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_43340cda5f2f434da15c0eed1e9ef675", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_f69971cc4dd64ff79d0d7351c12287f1", + "IPY_MODEL_3823997938bf40dfba99730e999d38cb" + ] + } + }, + "43340cda5f2f434da15c0eed1e9ef675": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "f69971cc4dd64ff79d0d7351c12287f1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_727bb31819804507b576ac2b328b3422", + "_dom_classes": [], + "description": "100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 348, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 348, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_261fa0d0628647c6a2312a611e14d9cb" + } + }, + "3823997938bf40dfba99730e999d38cb": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_692edef55b374430827d195cc3b251d7", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 348/348 [28:50<00:00, 4.97s/it]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_73290561972846cf98fa58c36371ab52" + } + }, + "727bb31819804507b576ac2b328b3422": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "261fa0d0628647c6a2312a611e14d9cb": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "692edef55b374430827d195cc3b251d7": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "73290561972846cf98fa58c36371ab52": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "433902e2a8f94aef8bce442e7f914128": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_327d4caa13f04666862911fb79f00f11", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_fb0ad282b4a94b67836a8eb0dbac1f15", + "IPY_MODEL_c229c34021f7472b959a8f7403912fe4" + ] + } + }, + "327d4caa13f04666862911fb79f00f11": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "fb0ad282b4a94b67836a8eb0dbac1f15": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_ce423ae5b69a4b0a97f007ebc80cc38c", + "_dom_classes": [], + "description": "100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 178, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 178, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_73c61abf85e046a093d712d1823a03ea" + } + }, + "c229c34021f7472b959a8f7403912fe4": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_7223671de4bd445fa1a012820f4127e0", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 178/178 [20:15<00:00, 6.83s/it]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_f7b05fbed307405281603766a5d4a58a" + } + }, + "ce423ae5b69a4b0a97f007ebc80cc38c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "73c61abf85e046a093d712d1823a03ea": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "7223671de4bd445fa1a012820f4127e0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "f7b05fbed307405281603766a5d4a58a": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "c904911b08f14f509c12ef27c615aeb7": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_1b256d0a93f043bea1518f2dfaf192b6", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_90e5c9b333364ef794f0cbc6adc32bf0", + "IPY_MODEL_a7b1a07f07da45aead68034595a4bd2c" + ] + } + }, + "1b256d0a93f043bea1518f2dfaf192b6": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "90e5c9b333364ef794f0cbc6adc32bf0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_6fc5a5a23b0c48318da07713129b7936", + "_dom_classes": [], + "description": "100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 354, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 354, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_8bd0605e7ebb44bd86238ea2ff980d52" + } + }, + "a7b1a07f07da45aead68034595a4bd2c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_e571040744c846afa7d53c2e591c3e7a", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 354/354 [11:06<00:00, 1.88s/it]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_d58e830f419244f4b8c77ca3f8b64a04" + } + }, + "6fc5a5a23b0c48318da07713129b7936": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "8bd0605e7ebb44bd86238ea2ff980d52": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "e571040744c846afa7d53c2e591c3e7a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "d58e830f419244f4b8c77ca3f8b64a04": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "64338d56af834b7f89d0cc42e4865979": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_91df941de0d648ee83aba211c5f471ed", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_5153db3076fb4ac2a7e0b8174355f1f1", + "IPY_MODEL_c1211d20730148dfbbeac83ca6728f76" + ] + } + }, + "91df941de0d648ee83aba211c5f471ed": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "5153db3076fb4ac2a7e0b8174355f1f1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_cc5bd3fd869d4293bb9636ce37205a7e", + "_dom_classes": [], + "description": "100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 178, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 178, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_01cc22d6b2634070ad47366cb5b02b01" + } + }, + "c1211d20730148dfbbeac83ca6728f76": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_1b1de2f4b4614c4ba898be7b66356119", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 178/178 [00:19<00:00, 9.32it/s]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_99b0ade69c3c4a468ec92a297210246a" + } + }, + "cc5bd3fd869d4293bb9636ce37205a7e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "01cc22d6b2634070ad47366cb5b02b01": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "1b1de2f4b4614c4ba898be7b66356119": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "99b0ade69c3c4a468ec92a297210246a": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "c32e43b5060242d4bfacf02024ea4359": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_66ab0def8d1c49a7b2ec4535d4b561bb", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_a36a6068d4c9421b932ff3d010cd1487", + "IPY_MODEL_1052294c4bfd433cac32f6b725b471ee" + ] + } + }, + "66ab0def8d1c49a7b2ec4535d4b561bb": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "a36a6068d4c9421b932ff3d010cd1487": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_9e82d6c3fbeb429281a12f0b68282022", + "_dom_classes": [], + "description": "100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 353, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 353, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_678716fa3bee44aaae2d48e0cd0a1e99" + } + }, + "1052294c4bfd433cac32f6b725b471ee": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_5fad3b5eb45f44ddb4611090acb7476c", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 353/353 [07:48<00:00, 1.33s/it]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_dc4e80d8f81a4a2db8903d1e65ca9b3e" + } + }, + "9e82d6c3fbeb429281a12f0b68282022": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "678716fa3bee44aaae2d48e0cd0a1e99": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "5fad3b5eb45f44ddb4611090acb7476c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "dc4e80d8f81a4a2db8903d1e65ca9b3e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "0a22ed02d5874441a37d48c6918ded61": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_8b9897b5418143dba6ce3391e25cee99", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_77637fe726054724bfe3dca8279f2d3a", + "IPY_MODEL_03203f66217f4f0bb0c8b01365947195" + ] + } + }, + "8b9897b5418143dba6ce3391e25cee99": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "77637fe726054724bfe3dca8279f2d3a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_6a163d3699c34e45b7f0020de518185a", + "_dom_classes": [], + "description": "100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 178, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 178, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_3128cba8d866450b8238eda0db1d31c0" + } + }, + "03203f66217f4f0bb0c8b01365947195": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_506f37817aba4731a3b63dd2eb43d9f2", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 178/178 [00:12<00:00, 14.71it/s]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_04e972f2ab334702b10829047d076f02" + } + }, + "6a163d3699c34e45b7f0020de518185a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "3128cba8d866450b8238eda0db1d31c0": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "506f37817aba4731a3b63dd2eb43d9f2": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "04e972f2ab334702b10829047d076f02": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + } + } + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Xrgq5Sa-Mjy9", + "colab_type": "text" + }, + "source": [ + "## BERTScore Tutorial" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Ogy116CAO0Ci", + "colab_type": "code", + "outputId": "7f9899d9-ffc5-48dc-d27f-2f96faf2f170", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 68 + } + }, + "source": [ + "!pip install torch" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (1.5.0+cu101)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torch) (1.18.4)\n", + "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch) (0.16.0)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "6cOBG7xyMjy_", + "colab_type": "code", + "outputId": "46b3023a-2182-449e-a21b-fa9bcbc7fdcc", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 734 + } + }, + "source": [ + "# if you have not installed bert_score, it is very easy\n", + "# simply uncomment the line below to install through pip\n", + "!pip install bert_score==0.2.2" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Collecting bert_score==0.2.2\n", + " Downloading https://files.pythonhosted.org/packages/47/d5/df44286c8bffc1e38730e889688b03a1a3c2575d673cb7028fd9b185f05b/bert_score-0.2.2-py3-none-any.whl\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from bert_score==0.2.2) (1.18.4)\n", + "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from bert_score==0.2.2) (3.2.1)\n", + "Requirement already satisfied: tqdm>=4.31.1 in /usr/local/lib/python3.6/dist-packages (from bert_score==0.2.2) (4.41.1)\n", + "Collecting transformers>=2.2.0\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/cd/38/c9527aa055241c66c4d785381eaf6f80a28c224cae97daa1f8b183b5fabb/transformers-2.9.0-py3-none-any.whl (635kB)\n", + "\u001b[K |████████████████████████████████| 645kB 4.6MB/s \n", + "\u001b[?25hRequirement already satisfied: torch>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from bert_score==0.2.2) (1.5.0+cu101)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from bert_score==0.2.2) (2.23.0)\n", + "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->bert_score==0.2.2) (2.4.7)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->bert_score==0.2.2) (0.10.0)\n", + "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->bert_score==0.2.2) (2.8.1)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->bert_score==0.2.2) (1.2.0)\n", + "Collecting sacremoses\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n", + "\u001b[K |████████████████████████████████| 890kB 17.4MB/s \n", + "\u001b[?25hCollecting tokenizers==0.7.0\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/14/e5/a26eb4716523808bb0a799fcfdceb6ebf77a18169d9591b2f46a9adb87d9/tokenizers-0.7.0-cp36-cp36m-manylinux1_x86_64.whl (3.8MB)\n", + "\u001b[K |████████████████████████████████| 3.8MB 21.5MB/s \n", + "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers>=2.2.0->bert_score==0.2.2) (3.0.12)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers>=2.2.0->bert_score==0.2.2) (2019.12.20)\n", + "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers>=2.2.0->bert_score==0.2.2) (0.7)\n", + "Collecting sentencepiece\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/3b/88/49e772d686088e1278766ad68a463513642a2a877487decbd691dec02955/sentencepiece-0.1.90-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n", + "\u001b[K |████████████████████████████████| 1.1MB 49.3MB/s \n", + "\u001b[?25hRequirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=1.0.0->bert_score==0.2.2) (0.16.0)\n", + "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->bert_score==0.2.2) (2.9)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->bert_score==0.2.2) (3.0.4)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->bert_score==0.2.2) (1.24.3)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->bert_score==0.2.2) (2020.4.5.1)\n", + "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from cycler>=0.10->matplotlib->bert_score==0.2.2) (1.12.0)\n", + "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.2.0->bert_score==0.2.2) (7.1.2)\n", + "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.2.0->bert_score==0.2.2) (0.14.1)\n", + "Building wheels for collected packages: sacremoses\n", + " Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=76172a6899e817fef3e62db089eb5c484a11b76fbf373b43528c43e7e07a31bd\n", + " Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n", + "Successfully built sacremoses\n", + "Installing collected packages: sacremoses, tokenizers, sentencepiece, transformers, bert-score\n", + "Successfully installed bert-score-0.2.2 sacremoses-0.0.43 sentencepiece-0.1.90 tokenizers-0.7.0 transformers-2.9.0\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bbove65oMjzC", + "colab_type": "text" + }, + "source": [ + "We will first demonstrate how to use the `score` function in `bert_score`, which is what you need to evaluate a set of machine generated outputs." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "b4z2xhj6MjzD", + "colab_type": "code", + "colab": {} + }, + "source": [ + "from bert_score import score" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "qYzDaQ7yNwht", + "colab_type": "code", + "outputId": "72376277-5bc2-4b57-cd34-c2aeb41233b8", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 136 + } + }, + "source": [ + "# cloning the repo because we need to get some example data\n", + "!git clone https://github.com/Tiiiger/bert_score.git" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Cloning into 'bert_score'...\n", + "remote: Enumerating objects: 141, done.\u001b[K\n", + "remote: Counting objects: 100% (141/141), done.\u001b[K\n", + "remote: Compressing objects: 100% (104/104), done.\u001b[K\n", + "remote: Total 605 (delta 72), reused 87 (delta 36), pack-reused 464\u001b[K\n", + "Receiving objects: 100% (605/605), 1.44 MiB | 11.25 MiB/s, done.\n", + "Resolving deltas: 100% (280/280), done.\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ARj1U5OsMjzI", + "colab_type": "text" + }, + "source": [ + "Inputs to `score` are a list of candidate sentences and a list of reference sentences. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ok21-9ZsMjzL", + "colab_type": "text" + }, + "source": [ + "Let's have a look." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "YYuDw0xgMjzM", + "colab_type": "code", + "outputId": "7ee71f37-895d-4730-ea5c-5531f33d9f11", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 122 + } + }, + "source": [ + "from google.colab import drive\n", + "drive.mount('/content/drive')" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n", + "\n", + "Enter your authorization code:\n", + "··········\n", + "Mounted at /content/drive\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "MtI2DzBUGh_d", + "colab_type": "code", + "colab": {} + }, + "source": [ + "import pandas as pd\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "4S8loHzFZEMJ", + "colab_type": "code", + "outputId": "fe8dd08b-48a0-4501-b951-fbd0b0a8821d", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 35 + } + }, + "source": [ + "! pwd" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/content\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "bwRX1vAcGnee", + "colab_type": "code", + "outputId": "b3bf1048-82a8-45e9-f4c5-b0d7da101ded", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 409 + } + }, + "source": [ + "####### This might give an error. Provide the path to the full_text data\n", + "\n", + "df2 = pd.read_csv('drive/My Drive/NLU Project/Data/full_text.txt')" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "error", + "ename": "FileNotFoundError", + "evalue": "ignored", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m####### This might give an error. Provide the path to the full_text data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mdf2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'drive/My Drive/NLU Project/Data/full_text.txt'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m 674\u001b[0m )\n\u001b[1;32m 675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 676\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 678\u001b[0m \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 446\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 447\u001b[0m \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 448\u001b[0;31m \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp_or_buf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 449\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 450\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m 878\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 879\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 880\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 881\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 882\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m 1112\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"c\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1113\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"c\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1114\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1115\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1116\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"python\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m 1889\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"usecols\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1890\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1891\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1892\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1893\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n", + "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] File drive/My Drive/NLU Project/Data/full_text.txt does not exist: 'drive/My Drive/NLU Project/Data/full_text.txt'" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "NNRJv4UfGqOD", + "colab_type": "code", + "colab": {} + }, + "source": [ + "refs = df2.text.to_list()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "a9aUXQNjHH8U", + "colab_type": "code", + "colab": {} + }, + "source": [ + "####### This might give an error. Provide the path to the model_predictions text_file\n", + "\n", + "df = pd.read_csv('/content/drive/My Drive/NLU Project/Data/model_predictions.txt')" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "cU8TyWjTIE3w", + "colab_type": "code", + "outputId": "f0e07cc2-2093-4183-bed7-e7f201884d63", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 289 + } + }, + "source": [ + "df.head()" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
indexgoldconvs2soracleleadptgentopic_convs2s
040453444on the first day in his new job , choe peng su...when hospitality choe , one of asia 's biggest...on a day-to-day basis mr choe says he still dr...fast forward about 20 years , and it 's fair t...it 's just two days since the start of the wor...when i first arrived in singapore , i did n't ...
136000053the women 's euro 2017 qualifier between north...the republic of ireland 's football team has b...`` the accident meant the motorway was closed ...`` the accident meant the motorway was closed ...northern ireland 's euro 2016 qualifier agains...the irish football association -lrb- uefa -rrb...
238711418theresa may is coming under pressure to say wh...theresa may has said she is `` very sorry '' a...questioned by andrew marr , the pm refused to ...the sunday times says the missile veered off c...scottish first minister nicola sturgeon has sa...prime minister theresa may has been accused of...
340452270us tennis star venus williams has been involve...a woman has died after being hit by a car in t...williams ' lawyer malcolm cunningham told cnn ...a spokesman for palm beach gardens police in f...a man who died after being hit by a car while ...us tennis star serena williams was involved in...
432472101ghana has been told by an international tribun...ghana 's supreme court has ruled that ghana 's...instead , it told the west african nation to `...the international tribunal for the law of the ...a tribunal has ruled that a number of ghana 's...the international tribunal in ghana has ruled ...
\n", + "
" + ], + "text/plain": [ + " index ... topic_convs2s\n", + "0 40453444 ... when i first arrived in singapore , i did n't ...\n", + "1 36000053 ... the irish football association -lrb- uefa -rrb...\n", + "2 38711418 ... prime minister theresa may has been accused of...\n", + "3 40452270 ... us tennis star serena williams was involved in...\n", + "4 32472101 ... the international tribunal in ghana has ruled ...\n", + "\n", + "[5 rows x 7 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 11 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "rZKOYKlJIG7V", + "colab_type": "code", + "colab": {} + }, + "source": [ + "cands_gold = df['gold'].to_list()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "al5f0NdZSh-I", + "colab_type": "code", + "outputId": "a6821ee5-bcbc-4d72-b8f2-3af2d1d21236", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 333, + "referenced_widgets": [ + "ea594def47344a209271cb3896a2829f", + "e681a632b25f42cd87a32aa278078743", + "b2a334d1247148b68f0f5d1ffddab2da", + "b0b91e181fa84d7194181b2d41093032", + "14815c9ba62b42a1b2b9aa5c232899e3", + "6b6ef2bab7c8479a8cf38b863f079829", + "9edeb2eb506f449d9dcf386bd3a95ac0", + "91b3c31e94474d31a5ecc0568093fc6d", + "ad048d2118aa4385800670defd5143b9", + "469b127ab1db46cca8dd856c3fcdfc9c", + "0e7cab84cb3541dba8d5f31de3ab3cb2", + "f718b658101147c999f9824e03fbaf61", + "6cc8f68307cb462baec3d8118b304fb3", + "39131ac8dff1423b9e1c2dcc1d1bc5bc", + "eec1c05633e7493bbbcbccac958aae52", + "21451cf863b3413bbc80e22f1377ad86", + "05d3f607ffc14acbb44853ec5962bacc", + "31aa05a6211d486d89c162119f6ca064", + "4a8656f24cc345299f23a958c66488fd", + "ef4dc6cfc73b4ee6a9636da04d6f0885", + "27fa5fd006a34745a017364859e5ca68", + "6b2189d4ed8e47bd80aebbcacb9eec62", + "6450eb81348b4ec4885f65c3bd0a0677", + "7451929a3c574b368ddb6b39333d9513", + "b170ede557524a73a32622fbe69818d6", + "4a45797271fe4e488441215249050c12", + "d49f5e320d5743da96526d5201d2cbf2", + "19757705fb034fa796afe99829dbf97c", + "6a7d84619c144f80913e10fa300f51ca", + "edfdcc7322d4482c88e3c505ad7f8817", + "07a8da43877c47e095d1727d233b70d0", + "3ceffa85777a4102a87a1a411c44ea41", + "50ecdf9b42844436b79e6ae8b4cf42e1", + "11ba2b7a69f148c8ba525a7b2f1a6306", + "e982c5477cd2437d81cdfa3ee45267dd", + "d3d16ce0315943729ce8cc31eff046b7", + "08f4ad3701aa43d9a1659e7f32173451", + "78fc3c361e3548559b9b8974bb3a5547", + "4cf5a5aa86e04f8da0955fc894095ada", + "1f4a0c088c2a49a2b10e28f88e67e35a" + ] + } + }, + "source": [ + "P, R, F1 = score(cands_gold, refs, model_type = 'bert-large-uncased', lang=\"en\", verbose=True)" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ea594def47344a209271cb3896a2829f", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=434.0, style=ProgressStyle(description_…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ad048d2118aa4385800670defd5143b9", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "05d3f607ffc14acbb44853ec5962bacc", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1344997306.0, style=ProgressStyle(descr…" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "calculating scores...\n", + "computing bert embedding.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b170ede557524a73a32622fbe69818d6", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=354.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "computing greedy matching.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "50ecdf9b42844436b79e6ae8b4cf42e1", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=178.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "done in 493.70 seconds, 22.96 sentences/sec\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "dGF1IzOTQye9", + "colab_type": "code", + "outputId": "1a8de76f-65f0-46db-c255-7570fc03aa6d", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + } + }, + "source": [ + "print(f\"System level F1 score: {F1.mean():.3f}\")\n", + "print(f\"System level P score: {P.mean():.3f}\")\n", + "print(f\"System level R score: {R.mean():.3f}\")\n" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "System level F1 score: 0.408\n", + "System level P score: 0.482\n", + "System level R score: 0.355\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "sII0WeqfQ_g1", + "colab_type": "code", + "colab": {} + }, + "source": [ + "del cands_gold" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "kcuKKPyYRBnJ", + "colab_type": "code", + "colab": {} + }, + "source": [ + "cands_convs2s = df['convs2s'].to_list()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "DflZA8QyREJY", + "colab_type": "code", + "outputId": "d698b94a-805c-42bb-ef87-bbf8b6eb3f52", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 185, + "referenced_widgets": [ + "3bf390f98f6e4782ab0779b87e99f8b1", + "3f2215159b68490ab50ebc97ec4bfc66", + "f241740a91a946118703b160c16ed74a", + "157a83855fca4d0a9c4b2a9c9774a56c", + "e2ac950e7f4649fca8ebd004b4d385e5", + "4e5c6954853a44c7a4697341f4c6a46d", + "d2f616ae80334e8cbb0a18d48d273df3", + "016afc9d449b434da2b49997d6951f3e", + "ebccc2638e1d4dd1a233cb86bc3ca08e", + "0f7098cf1dba49f8b6c2a65d20a57dc5", + "4c4564f0a6a74d76a9908e78b92be29f", + "2fe9f75b2ec44152ac4e1f0801b7e503", + "0d4d41f947bb4c85a773154e93390fe1", + "dbe54420e1074776892915293ead7724", + "43276d04629d484e8c1fb6b6318e894f", + "45d8217658d64e2bb842dda215f8ed1b" + ] + } + }, + "source": [ + "P, R, F1 = score(cands_convs2s, refs, model_type = 'bert-large-uncased', lang=\"en\", verbose=True)" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "calculating scores...\n", + "computing bert embedding.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3bf390f98f6e4782ab0779b87e99f8b1", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=353.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "computing greedy matching.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ebccc2638e1d4dd1a233cb86bc3ca08e", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=178.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "done in 483.37 seconds, 23.45 sentences/sec\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "ocr5IWuaRNOw", + "colab_type": "code", + "outputId": "45e15171-6f9f-427d-f150-810a11f0682c", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + } + }, + "source": [ + "print(f\"System level F1 score: {F1.mean():.3f}\")\n", + "print(f\"System level P score: {P.mean():.3f}\")\n", + "print(f\"System level R score: {R.mean():.3f}\")" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "System level F1 score: 0.408\n", + "System level P score: 0.493\n", + "System level R score: 0.349\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "hvxQnxLrRPPB", + "colab_type": "code", + "colab": {} + }, + "source": [ + "del cands_convs2s" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "S7aps9TSRRaY", + "colab_type": "code", + "colab": {} + }, + "source": [ + "cands_oracle = df['oracle'].to_list()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "a7dHTEgBRUFc", + "colab_type": "code", + "outputId": "c50fcbc3-3269-4ff1-bbab-69d363fe0118", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 185, + "referenced_widgets": [ + "ed55b90887f64449b12975722c797731", + "fa2b5c36f47a439998f38d4ce3f507d0", + "d866411994904eb58a0b275641410334", + "8736181c78d9418e9cbf6bbd049e17b0", + "ad8929a3d9104769a26892f0e8eb5ea5", + "a0605ad0cbdf4390a1f06320b53243c2", + "730b04fe7078495d9e938fcca577f116", + "cf86f43c2a004588adaafe0ff297d229", + "d91a7532623b46d497e5edd861025711", + "be94e5d8259743ce91dd7455e2d11cb0", + "b124ae1458374db28aa89336119ee7cc", + "32df47730e974a5eb56c40a948c83596", + "80507eb3237f4218a2a7b98ccd617fbf", + "9505637ab9f74f3d83ba71ae1683f1f0", + "7e3f6f793b2b4f0a9da89063e1344329", + "09248031d64b4612932e51f237a4fbb2" + ] + } + }, + "source": [ + "P, R, F1 = score(cands_oracle, refs, model_type = 'bert-large-uncased', lang=\"en\", verbose=True)" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "calculating scores...\n", + "computing bert embedding.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ed55b90887f64449b12975722c797731", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=354.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "computing greedy matching.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d91a7532623b46d497e5edd861025711", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=178.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "done in 489.43 seconds, 23.16 sentences/sec\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "eNFkUO2DRXKE", + "colab_type": "code", + "outputId": "7eb1ff34-e8a5-48b7-deaa-befe4a24dd5f", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + } + }, + "source": [ + "print(f\"System level F1 score: {F1.mean():.3f}\")\n", + "print(f\"System level P score: {P.mean():.3f}\")\n", + "print(f\"System level R score: {R.mean():.3f}\")" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "System level F1 score: 0.408\n", + "System level P score: 0.484\n", + "System level R score: 0.355\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "uHvVPk-nRlXd", + "colab_type": "code", + "colab": {} + }, + "source": [ + "del cands_oracle" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "RvMXu5eFReFl", + "colab_type": "code", + "colab": {} + }, + "source": [ + "cands_lead = df['lead'].to_list()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "hU03GB1NRgk3", + "colab_type": "code", + "outputId": "ef775ce7-c8f3-4dc8-81ef-540032c8dbbc", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 185, + "referenced_widgets": [ + "2ce06606f4fb4b7484a8531314f2fe03", + "43340cda5f2f434da15c0eed1e9ef675", + "f69971cc4dd64ff79d0d7351c12287f1", + "3823997938bf40dfba99730e999d38cb", + "727bb31819804507b576ac2b328b3422", + "261fa0d0628647c6a2312a611e14d9cb", + "692edef55b374430827d195cc3b251d7", + "73290561972846cf98fa58c36371ab52", + "433902e2a8f94aef8bce442e7f914128", + "327d4caa13f04666862911fb79f00f11", + "fb0ad282b4a94b67836a8eb0dbac1f15", + "c229c34021f7472b959a8f7403912fe4", + "ce423ae5b69a4b0a97f007ebc80cc38c", + "73c61abf85e046a093d712d1823a03ea", + "7223671de4bd445fa1a012820f4127e0", + "f7b05fbed307405281603766a5d4a58a" + ] + } + }, + "source": [ + "P, R, F1 = score(cands_lead, refs, model_type = 'bert-large-uncased', lang=\"en\", verbose=True)" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "calculating scores...\n", + "computing bert embedding.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2ce06606f4fb4b7484a8531314f2fe03", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=348.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "computing greedy matching.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "433902e2a8f94aef8bce442e7f914128", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=178.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "done in 488.55 seconds, 23.20 sentences/sec\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "8lrQnRYURu2Z", + "colab_type": "code", + "outputId": "a353f6a3-e6ee-46b0-cfe7-c998303d47ec", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + } + }, + "source": [ + "print(f\"System level F1 score: {F1.mean():.3f}\")\n", + "print(f\"System level P score: {P.mean():.3f}\")\n", + "print(f\"System level R score: {R.mean():.3f}\")" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "System level F1 score: 0.403\n", + "System level P score: 0.477\n", + "System level R score: 0.350\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Lfu06Zf6Rw9Q", + "colab_type": "code", + "colab": {} + }, + "source": [ + "del cands_lead" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "jFCuKJdfRqm9", + "colab_type": "code", + "colab": {} + }, + "source": [ + "cands_ptgen = df['ptgen'].to_list()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "qS-P4dejRzBS", + "colab_type": "code", + "outputId": "93caf4a1-6823-46e6-eb57-f620319ae8cf", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 185, + "referenced_widgets": [ + "c904911b08f14f509c12ef27c615aeb7", + "1b256d0a93f043bea1518f2dfaf192b6", + "90e5c9b333364ef794f0cbc6adc32bf0", + "a7b1a07f07da45aead68034595a4bd2c", + "6fc5a5a23b0c48318da07713129b7936", + "8bd0605e7ebb44bd86238ea2ff980d52", + "e571040744c846afa7d53c2e591c3e7a", + "d58e830f419244f4b8c77ca3f8b64a04", + "64338d56af834b7f89d0cc42e4865979", + "91df941de0d648ee83aba211c5f471ed", + "5153db3076fb4ac2a7e0b8174355f1f1", + "c1211d20730148dfbbeac83ca6728f76", + "cc5bd3fd869d4293bb9636ce37205a7e", + "01cc22d6b2634070ad47366cb5b02b01", + "1b1de2f4b4614c4ba898be7b66356119", + "99b0ade69c3c4a468ec92a297210246a" + ] + } + }, + "source": [ + "P, R, F1 = score(cands_ptgen, refs, model_type = 'bert-large-uncased', lang=\"en\", verbose=True)" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "calculating scores...\n", + "computing bert embedding.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c904911b08f14f509c12ef27c615aeb7", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=354.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "computing greedy matching.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "64338d56af834b7f89d0cc42e4865979", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=178.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "done in 484.33 seconds, 23.40 sentences/sec\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "2kHxT0GbR4L7", + "colab_type": "code", + "outputId": "2596e311-eb37-453c-f00e-bdd4cc14d21c", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + } + }, + "source": [ + "print(f\"System level F1 score: {F1.mean():.3f}\")\n", + "print(f\"System level P score: {P.mean():.3f}\")\n", + "print(f\"System level R score: {R.mean():.3f}\")" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "System level F1 score: 0.410\n", + "System level P score: 0.489\n", + "System level R score: 0.354\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "LdBJUipNR7uQ", + "colab_type": "code", + "colab": {} + }, + "source": [ + "del cands_ptgen" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "4-b2mBqQITqA", + "colab_type": "code", + "colab": {} + }, + "source": [ + "cands_topic_convs2s = df['topic_convs2s'].to_list()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "b6PJ9Z1xG6lT", + "colab_type": "code", + "outputId": "eb5f748d-3a65-4c4d-a8c9-a6b44c92f15b", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 185, + "referenced_widgets": [ + "c32e43b5060242d4bfacf02024ea4359", + "66ab0def8d1c49a7b2ec4535d4b561bb", + "a36a6068d4c9421b932ff3d010cd1487", + "1052294c4bfd433cac32f6b725b471ee", + "9e82d6c3fbeb429281a12f0b68282022", + "678716fa3bee44aaae2d48e0cd0a1e99", + "5fad3b5eb45f44ddb4611090acb7476c", + "dc4e80d8f81a4a2db8903d1e65ca9b3e", + "0a22ed02d5874441a37d48c6918ded61", + "8b9897b5418143dba6ce3391e25cee99", + "77637fe726054724bfe3dca8279f2d3a", + "03203f66217f4f0bb0c8b01365947195", + "6a163d3699c34e45b7f0020de518185a", + "3128cba8d866450b8238eda0db1d31c0", + "506f37817aba4731a3b63dd2eb43d9f2", + "04e972f2ab334702b10829047d076f02" + ] + } + }, + "source": [ + "P, R, F1 = score(cands_topic_convs2s, refs, model_type = 'bert-large-uncased', lang=\"en\", verbose=True)" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "calculating scores...\n", + "computing bert embedding.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c32e43b5060242d4bfacf02024ea4359", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=353.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "computing greedy matching.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0a22ed02d5874441a37d48c6918ded61", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=178.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\n", + "done in 482.44 seconds, 23.49 sentences/sec\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "BIJHXn4oSGX3", + "colab_type": "code", + "outputId": "d1f19de2-6c2e-4880-f1da-69b7377ecf8c", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + } + }, + "source": [ + "print(f\"System level F1 score: {F1.mean():.3f}\")\n", + "print(f\"System level P score: {P.mean():.3f}\")\n", + "print(f\"System level R score: {R.mean():.3f}\")" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "System level F1 score: 0.407\n", + "System level P score: 0.492\n", + "System level R score: 0.349\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "BFndA87gXx-S", + "colab_type": "code", + "colab": {} + }, + "source": [ + "" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/ROUGE_score_calculator.ipynb b/ROUGE_score_calculator.ipynb new file mode 100644 index 0000000..fd0921b --- /dev/null +++ b/ROUGE_score_calculator.ipynb @@ -0,0 +1,245 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "ROUGE_score_calculator.ipynb", + "provenance": [], + "collapsed_sections": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + } + }, + "cells": [ + { + "cell_type": "code", + "metadata": { + "id": "WfEFBXZvAbK3", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 102 + }, + "outputId": "16ee043e-a756-4f58-a32c-ec97cb0f9560" + }, + "source": [ + "!pip install rouge\n", + "from rouge import Rouge " + ], + "execution_count": 1, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Collecting rouge\n", + " Downloading https://files.pythonhosted.org/packages/43/cc/e18e33be20971ff73a056ebdb023476b5a545e744e3fc22acd8c758f1e0d/rouge-1.0.0-py3-none-any.whl\n", + "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from rouge) (1.12.0)\n", + "Installing collected packages: rouge\n", + "Successfully installed rouge-1.0.0\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "JJqnU9LjAzIN", + "colab_type": "code", + "colab": {} + }, + "source": [ + "import sys\n", + "sys.setrecursionlimit(15000)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "zyUqUDViBc3l", + "colab_type": "code", + "colab": {} + }, + "source": [ + "import pandas as pd" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "oluhyd5_BGvc", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 122 + }, + "outputId": "8011e616-9571-4972-af1c-1c64d0cd276f" + }, + "source": [ + "from google.colab import drive\n", + "drive.mount('/content/drive')" + ], + "execution_count": 3, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n", + "\n", + "Enter your authorization code:\n", + "··········\n", + "Mounted at /content/drive\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "h9Nz5g-dA0Tt", + "colab_type": "code", + "colab": {} + }, + "source": [ + "data = pd.read_csv('/content/drive/My Drive/pfull_text.txt')\n", + "data_hyps = pd.read_csv('/content/drive/My Drive/model_predictions.txt')" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "rTVofq5eArwo", + "colab_type": "code", + "colab": {} + }, + "source": [ + "hyps = data_hyps['oracle']\n", + "refs = data['text']\n", + "rouge = Rouge()\n", + "scores = rouge.get_scores(hyps, refs, avg=True)\n", + "print(scores)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Weu-_16Yp52o", + "colab_type": "code", + "colab": {} + }, + "source": [ + "hyps = data_hyps['gold']\n", + "refs = data['text']\n", + "rouge = Rouge()\n", + "scores = rouge.get_scores(hyps, refs, avg=True)\n", + "print(scores)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "2bYqQrZ5AgW9", + "colab_type": "code", + "colab": {} + }, + "source": [ + "hyps = data_hyps['lead']\n", + "refs = data['text']\n", + "rouge = Rouge()\n", + "scores = rouge.get_scores(hyps, refs, avg=True)\n", + "print(scores)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "QIp7zwCfBg0r", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 170 + }, + "outputId": "9ab8d856-c75e-474c-cafe-888738e13500" + }, + "source": [ + "scoreshyps = data_hyps['convs2s']\n", + "refs = data['text']\n", + "rouge = Rouge()\n", + "scores = rouge.get_scores(hyps, refs, avg=True)\n", + "print(scores)" + ], + "execution_count": 10, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "{'rouge-1': {'f': 0.05604884526638604,\n", + " 'p': 0.37793766330603457,\n", + " 'r': 0.03257953463894991},\n", + " 'rouge-2': {'f': 0.0046201542993716095,\n", + " 'p': 0.039091481737943176,\n", + " 'r': 0.002592478990316501},\n", + " 'rouge-l': {'f': 0.06409551384670242,\n", + " 'p': 0.3132453970743527,\n", + " 'r': 0.03779617006573437}}" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 10 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "uld3woPcB-j6", + "colab_type": "code", + "colab": {} + }, + "source": [ + "hyps = data_hyps['topic_convs2s']\n", + "refs = data['text']\n", + "rouge = Rouge()\n", + "scores = rouge.get_scores(hyps, refs, avg=True)\n", + "print(scores)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "dgNTKdQ_qCpo", + "colab_type": "code", + "colab": {} + }, + "source": [ + "hyps = data_hyps['ptgen']\n", + "refs = data['text']\n", + "rouge = Rouge()\n", + "scores = rouge.get_scores(hyps, refs, avg=True)\n", + "print(scores)" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/Sentence_Bert_Score_calculator.ipynb b/Sentence_Bert_Score_calculator.ipynb new file mode 100644 index 0000000..b52db68 --- /dev/null +++ b/Sentence_Bert_Score_calculator.ipynb @@ -0,0 +1,267 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "Sentence_Bert_Score_calculator.ipynb", + "provenance": [], + "machine_shape": "hm" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + } + }, + "cells": [ + { + "cell_type": "code", + "metadata": { + "id": "ZT3kr4CezYkg", + "colab_type": "code", + "outputId": "7578e53b-6471-4627-a9b2-4499531e3a60", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 122 + } + }, + "source": [ + "from google.colab import drive\n", + "drive.mount('/content/drive')" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n", + "\n", + "Enter your authorization code:\n", + "··········\n", + "Mounted at /content/drive\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Of4T_FM4zsl4", + "colab_type": "code", + "colab": {} + }, + "source": [ + "import pandas as pd\n", + "import numpy as np \n", + "from sklearn.metrics.pairwise import cosine_similarity" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "QF8b_UOBHE9a", + "colab_type": "code", + "colab": {} + }, + "source": [ + "def cosine_sim(z,praxal):\n", + " return np.sum(np.multiply(praxal,z),axis=1)/np.multiply(np.linalg.norm(z,axis=1),np.linalg.norm(praxal,axis=1))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "mbinlshTz1fL", + "colab_type": "code", + "colab": {} + }, + "source": [ + "full_text_emb = np.load('/content/drive/My Drive/Data/sentence-embeddings/full_text_sent_embeddings.csv.npy')" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "RBe9GWAazL3S", + "colab_type": "code", + "outputId": "3480944b-817d-4285-c546-4355e792dc9f", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + } + }, + "source": [ + "gold_emb = np.load('/content/drive/My Drive/Data/sentence-embeddings/gold_sent_embeddings.csv.npy')\n", + "a = cosine_sim(gold_emb, full_text_emb)\n", + "print (np.mean(a))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.48500752\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GVXXjoBS0Ann", + "colab_type": "code", + "outputId": "d0d129b8-1086-41ac-eb4d-6b4aee4dad6d", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + } + }, + "source": [ + "lead_emb = np.load('/content/drive/My Drive/Data/sentence-embeddings/lead_sent_embeddings.csv.npy')\n", + "\n", + "a = cosine_sim(lead_emb, full_text_emb)\n", + "print (np.mean(a))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.44751462\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "6UJHGI8c0QCw", + "colab_type": "code", + "outputId": "62bb002a-d326-4db4-f692-e7e23815d759", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + } + }, + "source": [ + "oracle_emb = np.load('/content/drive/My Drive/Data/sentence-embeddings/oracle_sent_embeddings.csv.npy')\n", + "a = cosine_sim(oracle_emb,full_text_emb)\n", + "print (np.mean(a))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.43796578\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GGGqKCVQ0hQ-", + "colab_type": "code", + "outputId": "1d2c1cfb-3467-46ac-cc12-b865ec226101", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + } + }, + "source": [ + "topic_conv_emb = np.load('/content/drive/My Drive/Data/sentence-embeddings/topic_sent_embeddings.csv.npy')\n", + "a = cosine_sim(topic_conv_emb,full_text_emb)\n", + "print (np.mean(a))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.44146976\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "gnU3-RNC02Ns", + "colab_type": "code", + "outputId": "a9546702-aa06-4e95-a71b-74cffa1ec637", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + } + }, + "source": [ + "pptgen_emb = np.load('/content/drive/My Drive/Data/sentence-embeddings/ptgen_sent_embeddings.csv.npy')\n", + "\n", + "a = cosine_sim(pptgen_emb,full_text_emb)\n", + "print (np.mean(a))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.46474588\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "BF96f04m1IOH", + "colab_type": "code", + "outputId": "be4d2257-32b4-45de-ed7e-5c81f961881d", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + } + }, + "source": [ + "conv_emb = np.load('/content/drive/My Drive/Data/sentence-embeddings/conv2s_std_sent_embeddings.csv.npy')\n", + "\n", + "a = cosine_sim(conv_emb,full_text_emb)\n", + "print (np.mean(a))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.43901289\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "xPi_yQ7y5I5L", + "colab_type": "code", + "colab": {} + }, + "source": [ + "" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/Sentence_Embedding.ipynb b/Sentence_Embedding.ipynb new file mode 100644 index 0000000..5eb0044 --- /dev/null +++ b/Sentence_Embedding.ipynb @@ -0,0 +1,1148 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 122 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 20623, + "status": "ok", + "timestamp": 1589584819642, + "user": { + "displayName": "Parthvi Shah", + "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj14forhnYAb8tD0SHMMufGgXmEIpTA6BXAnPZ5Uvg=s64", + "userId": "12920190590086167612" + }, + "user_tz": 240 + }, + "id": "1IJI3jL1r9k6", + "outputId": "eb300f85-0a5e-4ff8-db9b-903ad0ad1449" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n", + "\n", + "Enter your authorization code:\n", + "··········\n", + "Mounted at /content/drive\n" + ] + } + ], + "source": [ + "from google.colab import drive\n", + "drive.mount('/content/drive')" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "nLtnI7rh0kHV" + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "df2 = pd.read_csv('/content/drive/My Drive/Data/full_text.txt')" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 623 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 1384, + "status": "ok", + "timestamp": 1589584840495, + "user": { + "displayName": "Parthvi Shah", + "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj14forhnYAb8tD0SHMMufGgXmEIpTA6BXAnPZ5Uvg=s64", + "userId": "12920190590086167612" + }, + "user_tz": 240 + }, + "id": "5ZXVp22M0uN3", + "outputId": "9cca4a1a-9c21-4a2a-cb08-c1dcb64bc098" + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
indexgoldconvs2soracleleadptgentopic_convs2s
873110001244n-dubz have told newsbeat they are shocked to ...london trio and we got are among the nominees ...n-dubz said they were looking forward to perfo...the london trio are up for best uk act and bes...the scottish fans have won the best song prize...one direction have been nominated for this yea...
645510138849`` maverick '' is a word that seems to follow ...dr robert venter , who has died at the age of ...and he is a very wealthy user of lear jets and...the biologist and entrepreneur turned the effo...dr dr [UNK] venter looks set to become a docto...scientists in vietnam have used the nobel peac...
622310145459australia has expelled an israeli diplomat say...israel has said it is `` disappointed '' by is...at least four forged australian passports were...australia 's foreign minister said these were ...australia has condemned the killing of israeli...israel has refused to apologise to the israeli...
338910169745bbc world affairs correspondent peter biles ha...one of the world 's most famous sailors , for ...for 70 years , my father has lived with the sh...for 70 years , my father has lived with the sh...on the surface of dunkirk a day , the hebe of ...the world 's oldest ship , the ship , has died...
1078410201300a maoist leader in india has said that they wi...the chief minister of india 's tamil nadu stat...report said other services were being reschedu...comrade akaash 's statement comes after the re...india 's railways has said it will not investi...the chief minister of the indian state of utta...
........................
557841088404theresa may has said china must do more to hel...japanese prime minister shinzo abe has said th...japan is preoccupied with the issue of north k...on a visit to japan , the prime minister welco...north korea has said it is ready to invest $ 1...japanese prime minister theresa may has said t...
714541089216england fly-half owen farrell has signed a new...saracens fly-half owen farrell has signed a ne...farrell has 52 caps for england and four for t...the 25-year-old , who has helped sarries to th...saracens forward owen farrell has signed a new...saracens and england fly-half owen farrell has...
316641091477prince william and prince harry have visited a...the duke and duchess of cambridge have visited...members of the public have been leaving tribut...the white garden , at kensington palace , was ...a memorial to remember the duchess of cambridg...the duke and duchess of cambridge have visited...
127541094254a judge in the case of a five-year-old christi...the number of children in foster care in engla...the local authority involved , the london boro...the local authority involved , the london boro...the family of a vulnerable child in tower haml...the number of foster children in foster care i...
354641141109mercedes ' lewis hamilton took the outright ch...lewis hamilton beat mercedes team-mate lewis h...hamilton 's victory was among the most straigh...a day after breaking the all-time record for p...nico vettel set the pace in second practice at...ferrari 's sebastian vettel set the pace in se...
\n", + "

11334 rows × 7 columns

\n", + "
" + ], + "text/plain": [ + " index ... topic_convs2s\n", + "8731 10001244 ... one direction have been nominated for this yea...\n", + "6455 10138849 ... scientists in vietnam have used the nobel peac...\n", + "6223 10145459 ... israel has refused to apologise to the israeli...\n", + "3389 10169745 ... the world 's oldest ship , the ship , has died...\n", + "10784 10201300 ... the chief minister of the indian state of utta...\n", + "... ... ... ...\n", + "5578 41088404 ... japanese prime minister theresa may has said t...\n", + "7145 41089216 ... saracens and england fly-half owen farrell has...\n", + "3166 41091477 ... the duke and duchess of cambridge have visited...\n", + "1275 41094254 ... the number of foster children in foster care i...\n", + "3546 41141109 ... ferrari 's sebastian vettel set the pace in se...\n", + "\n", + "[11334 rows x 7 columns]" + ] + }, + "execution_count": 4, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "model_preds = pd.read_csv('/content/drive/My Drive/Data/model_predictions.txt')\n", + "model_preds.sort_values('index')" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "7szXZCRZkJjj" + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "mpmsyFWXtink" + }, + "outputs": [], + "source": [ + "df2list = df2.text.to_list()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 54 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 348, + "status": "ok", + "timestamp": 1589584843389, + "user": { + "displayName": "Parthvi Shah", + "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj14forhnYAb8tD0SHMMufGgXmEIpTA6BXAnPZ5Uvg=s64", + "userId": "12920190590086167612" + }, + "user_tz": 240 + }, + "id": "WDyu1syuwhXo", + "outputId": "76fe5aea-685b-4ccd-f221-bcf6082aa6ad" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "\"The London trio are up for best UK act and best album , as well as getting two nominations in the best song category .\\n`` We got told like this morning ` Oh I think you 're nominated ' '' , said Dappy .\\n`` And I was like ` Oh yeah , which one ? '\\nAnd now we 've got nominated for four awards .\\nI mean , wow ! ''\\nBandmate Fazer added : `` We thought it 's best of us to come down and mingle with everyone and say hello to the cameras .\\nAnd now we find we 've got four nominations . ''\\nThe band have two shots at the best song prize , getting the nod for their Tynchy Stryder collaboration Number One , and single Strong Again .\\nTheir album Uncle B will also go up against records by the likes of Beyonce and Kanye West .\\nN-Dubz picked up the best newcomer Mobo in 2007 , but female member Tulisa said they would n't be too disappointed if they did n't win this time around .\\n`` At the end of the day we 're grateful to be where we are in our careers .\\n`` If it do n't happen then it do n't happen - live to fight another day and keep on making albums and hits for the fans . ''\\nDappy also revealed they could be performing live several times on the night .\\nThe group will be doing Number One and also a possible rendition of the War Child single , I Got Soul .\\nThe charity song is a re-working of The Killers ' All These Things That I 've Done and is set to feature artists like Chipmunk , Ironik and Pixie Lott .\\nThis year 's Mobos will be held outside of London for the first time , in Glasgow on 30 September .\\nN-Dubz said they were looking forward to performing for their Scottish fans and boasted about their recent shows north of the border .\\n`` We just done Edinburgh the other day , '' said Dappy .\\n`` We smashed up an N-Dubz show over there .\\nWe done Aberdeen about three or four months ago - we smashed up that show over there !\\nEverywhere we go we smash it up ! ''\\n\"" + ] + }, + "execution_count": 6, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "df2list[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 768 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 7906, + "status": "ok", + "timestamp": 1589584852392, + "user": { + "displayName": "Parthvi Shah", + "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj14forhnYAb8tD0SHMMufGgXmEIpTA6BXAnPZ5Uvg=s64", + "userId": "12920190590086167612" + }, + "user_tz": 240 + }, + "id": "mCgv9zKos-VM", + "outputId": "7dd2f54c-a59b-4cf3-ce8c-edea3704fa6b" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting sentence-transformers\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/b9/46/b7d6c37d92d1bd65319220beabe4df845434930e3f30e42d3cfaecb74dc4/sentence-transformers-0.2.6.1.tar.gz (55kB)\n", + "\u001b[K |████████████████████████████████| 61kB 2.3MB/s \n", + "\u001b[?25hCollecting transformers>=2.8.0\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/22/97/7db72a0beef1825f82188a4b923e62a146271ac2ced7928baa4d47ef2467/transformers-2.9.1-py3-none-any.whl (641kB)\n", + "\u001b[K |████████████████████████████████| 645kB 8.5MB/s \n", + "\u001b[?25hRequirement already satisfied, skipping upgrade: tqdm in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (4.41.1)\n", + "Requirement already satisfied, skipping upgrade: torch>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (1.5.0+cu101)\n", + "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (1.18.4)\n", + "Requirement already satisfied, skipping upgrade: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (0.22.2.post1)\n", + "Requirement already satisfied, skipping upgrade: scipy in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (1.4.1)\n", + "Requirement already satisfied, skipping upgrade: nltk in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (3.2.5)\n", + "Collecting tokenizers==0.7.0\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/14/e5/a26eb4716523808bb0a799fcfdceb6ebf77a18169d9591b2f46a9adb87d9/tokenizers-0.7.0-cp36-cp36m-manylinux1_x86_64.whl (3.8MB)\n", + "\u001b[K |████████████████████████████████| 3.8MB 16.6MB/s \n", + "\u001b[?25hRequirement already satisfied, skipping upgrade: filelock in /usr/local/lib/python3.6/dist-packages (from transformers>=2.8.0->sentence-transformers) (3.0.12)\n", + "Collecting sentencepiece\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/3b/88/49e772d686088e1278766ad68a463513642a2a877487decbd691dec02955/sentencepiece-0.1.90-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n", + "\u001b[K |████████████████████████████████| 1.1MB 46.5MB/s \n", + "\u001b[?25hCollecting sacremoses\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n", + "\u001b[K |████████████████████████████████| 890kB 57.5MB/s \n", + "\u001b[?25hRequirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.6/dist-packages (from transformers>=2.8.0->sentence-transformers) (2.23.0)\n", + "Requirement already satisfied, skipping upgrade: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers>=2.8.0->sentence-transformers) (2019.12.20)\n", + "Requirement already satisfied, skipping upgrade: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers>=2.8.0->sentence-transformers) (0.7)\n", + "Requirement already satisfied, skipping upgrade: future in /usr/local/lib/python3.6/dist-packages (from torch>=1.0.1->sentence-transformers) (0.16.0)\n", + "Requirement already satisfied, skipping upgrade: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sentence-transformers) (0.14.1)\n", + "Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from nltk->sentence-transformers) (1.12.0)\n", + "Requirement already satisfied, skipping upgrade: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.8.0->sentence-transformers) (7.1.2)\n", + "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.8.0->sentence-transformers) (2020.4.5.1)\n", + "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.8.0->sentence-transformers) (3.0.4)\n", + "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.8.0->sentence-transformers) (1.24.3)\n", + "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.8.0->sentence-transformers) (2.9)\n", + "Building wheels for collected packages: sentence-transformers, sacremoses\n", + " Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for sentence-transformers: filename=sentence_transformers-0.2.6.1-cp36-none-any.whl size=74031 sha256=18db97fe6600e5eb2b71cfe5efcd81fa43a8f63d6ebbbdbe0c91c1714d064fc8\n", + " Stored in directory: /root/.cache/pip/wheels/d7/fa/17/2b081a8cd8b0a86753fb0e9826b3cc19f0207062c0b2da7008\n", + " Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=0aba317fc1e6f9a45122363c0a0a543b33aa78d420d4db15d597e86ecdebc531\n", + " Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n", + "Successfully built sentence-transformers sacremoses\n", + "Installing collected packages: tokenizers, sentencepiece, sacremoses, transformers, sentence-transformers\n", + "Successfully installed sacremoses-0.0.43 sentence-transformers-0.2.6.1 sentencepiece-0.1.90 tokenizers-0.7.0 transformers-2.9.1\n" + ] + } + ], + "source": [ + "!pip install -U sentence-transformers\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 32953, + "status": "ok", + "timestamp": 1589584886230, + "user": { + "displayName": "Parthvi Shah", + "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj14forhnYAb8tD0SHMMufGgXmEIpTA6BXAnPZ5Uvg=s64", + "userId": "12920190590086167612" + }, + "user_tz": 240 + }, + "id": "g-yJRBSXtQUt", + "outputId": "192c3177-fb23-47ae-ca79-8099f0112f21" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 405M/405M [00:20<00:00, 19.5MB/s]\n" + ] + } + ], + "source": [ + "from sentence_transformers import SentenceTransformer\n", + "model = SentenceTransformer('bert-base-nli-mean-tokens', max_seq_length = 800)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "VblG434jXe4C" + }, + "outputs": [], + "source": [ + "sentence_embeddings = model.encode(df2list)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 465, + "status": "ok", + "timestamp": 1588694037856, + "user": { + "displayName": "Parthvi Shah", + "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj14forhnYAb8tD0SHMMufGgXmEIpTA6BXAnPZ5Uvg=s64", + "userId": "12920190590086167612" + }, + "user_tz": 240 + }, + "id": "1LoQpbmMio57", + "outputId": "a4200e1c-5592-4100-acc2-67712ecf3e7e" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sentence: The London trio are up for best UK act and best album , as well as getting two nominations in the best song category .\n", + "`` We got told like this morning ` Oh I think you 're nominated ' '' , said Dappy .\n", + "`` And I was like ` Oh yeah , which one ? '\n", + "And now we 've got nominated for four awards .\n", + "I mean , wow ! ''\n", + "Bandmate Fazer added : `` We thought it 's best of us to come down and mingle with everyone and say hello to the cameras .\n", + "And now we find we 've got four nominations . ''\n", + "The band have two shots at the best song prize , getting the nod for their Tynchy Stryder collaboration Number One , and single Strong Again .\n", + "Their album Uncle B will also go up against records by the likes of Beyonce and Kanye West .\n", + "N-Dubz picked up the best newcomer Mobo in 2007 , but female member Tulisa said they would n't be too disappointed if they did n't win this time around .\n", + "`` At the end of the day we 're grateful to be where we are in our careers .\n", + "`` If it do n't happen then it do n't happen - live to fight another day and keep on making albums and hits for the fans . ''\n", + "Dappy also revealed they could be performing live several times on the night .\n", + "The group will be doing Number One and also a possible rendition of the War Child single , I Got Soul .\n", + "The charity song is a re-working of The Killers ' All These Things That I 've Done and is set to feature artists like Chipmunk , Ironik and Pixie Lott .\n", + "This year 's Mobos will be held outside of London for the first time , in Glasgow on 30 September .\n", + "N-Dubz said they were looking forward to performing for their Scottish fans and boasted about their recent shows north of the border .\n", + "`` We just done Edinburgh the other day , '' said Dappy .\n", + "`` We smashed up an N-Dubz show over there .\n", + "We done Aberdeen about three or four months ago - we smashed up that show over there !\n", + "Everywhere we go we smash it up ! ''\n", + "\n", + "Embedding: [-1.70456335e-01 1.17136002e+00 1.25976241e+00 -3.49129349e-01\n", + " -5.59076011e-01 -6.66664243e-01 8.57612967e-01 -1.89061597e-01\n", + " -3.01911915e-03 3.89481336e-01 1.68308333e-01 9.55304205e-01\n", + " 6.29280508e-01 3.73455048e-01 -1.11567557e+00 9.30601478e-01\n", + " -1.83928192e-01 -1.01337336e-01 -1.88596457e-01 -3.81882697e-01\n", + " 2.15971977e-01 -3.42547208e-01 2.16453761e-01 -3.81739601e-03\n", + " 1.28886378e+00 1.09037411e+00 -3.92858714e-01 5.41742854e-02\n", + " 4.56132919e-01 1.87388226e-01 2.71019638e-01 4.79159653e-01\n", + " -5.66693962e-01 -3.77531767e-01 -5.62979698e-01 4.36255932e-01\n", + " 2.99058944e-01 -2.09565815e-02 3.47341746e-01 -1.99638247e-01\n", + " 5.22924423e-01 -3.31639171e-01 -1.31694842e-02 -3.39082837e-01\n", + " -1.42144990e+00 -5.81485391e-01 -7.70625249e-02 9.61425960e-01\n", + " 6.58001184e-01 -8.98526847e-01 -1.24103166e-01 9.24842712e-03\n", + " -8.33332896e-01 -4.57403362e-01 -5.85393369e-01 -5.41878879e-01\n", + " -1.37695745e-01 -6.43214703e-01 -8.92668188e-01 -6.42860889e-01\n", + " -1.47272456e+00 9.75339264e-02 -6.76050127e-01 -3.29256564e-01\n", + " 1.77658461e-02 -2.66443081e-02 3.57944936e-01 6.48705542e-01\n", + " -7.54494667e-01 -3.52582335e-01 -4.99034166e-01 -2.48280510e-01\n", + " 7.17171431e-02 -5.57472706e-01 -4.53067534e-02 6.77652359e-02\n", + " 4.12348323e-02 -2.40977481e-01 1.10805178e+00 1.01880884e+00\n", + " 4.93538558e-01 -2.26209879e-01 -2.31551707e-01 4.58131939e-01\n", + " -3.84624079e-02 2.54799515e-01 1.86189532e-01 -8.86694789e-01\n", + " 2.57310390e-01 6.75687790e-01 3.64805758e-01 -5.77559829e-01\n", + " 1.32404447e+00 -1.82508141e-01 -9.33061302e-01 -4.09886330e-01\n", + " 1.00693323e-01 -1.01789820e+00 -4.79020685e-01 -1.37053937e-01\n", + " 5.50992861e-02 -3.96825075e-01 -3.31854016e-01 3.12445670e-01\n", + " -5.43893635e-01 2.68868446e-01 -2.39529699e-01 -1.02851391e+00\n", + " -1.05420418e-01 1.89755842e-01 1.17458403e+00 -5.23348749e-01\n", + " -6.62774742e-01 2.76699930e-01 6.02474451e-01 -9.81754601e-01\n", + " -9.77740109e-01 6.33293569e-01 -2.17767134e-02 2.73366392e-01\n", + " 7.93746769e-01 -4.34435159e-02 -5.86929798e-01 -3.14657204e-02\n", + " 3.59440118e-01 -1.69528909e-02 1.18122689e-01 9.10280049e-01\n", + " -3.16781074e-01 8.29022937e-03 -1.34455010e-01 5.48023462e-01\n", + " 6.40620589e-01 -1.41183019e-01 2.78975572e-02 6.33963704e-01\n", + " 3.80491763e-02 -2.71837205e-01 -4.57520485e-01 -1.46897927e-01\n", + " -6.99306233e-03 -7.00604200e-01 3.47114801e-01 -6.52771831e-01\n", + " 4.39300001e-01 4.86489087e-02 -6.01151064e-02 2.71719526e-02\n", + " 4.41433452e-02 -2.69283861e-01 5.17906249e-01 1.82797536e-02\n", + " -1.31894898e+00 5.28349653e-02 -3.08292210e-01 -5.60247362e-01\n", + " -3.44262123e-01 -4.64371979e-01 2.64644414e-01 -2.66881764e-01\n", + " 2.67573595e-01 -2.25550141e-02 2.97785103e-01 1.72345769e+00\n", + " 5.87788582e-01 -6.14552021e-01 6.04319394e-01 -5.99832647e-02\n", + " -2.35066079e-02 -2.72660498e-02 -3.40155274e-01 6.11551642e-01\n", + " 1.21816635e+00 5.69462240e-01 1.05885553e+00 -9.54238176e-02\n", + " 5.92006207e-01 3.66368532e-01 -8.45839024e-01 3.94693851e-01\n", + " 2.35582620e-01 6.91884518e-01 -1.86187887e+00 9.57286477e-01\n", + " 5.65307438e-01 -4.90228802e-01 8.41324270e-01 -2.48000652e-01\n", + " -8.31936300e-02 -1.19295567e-01 -5.59261858e-01 -8.68661702e-02\n", + " -6.75781369e-02 5.90043254e-02 5.98990098e-02 3.36321294e-01\n", + " 5.88541254e-02 8.57590318e-01 2.25650132e-01 9.59640443e-01\n", + " -4.71969783e-01 6.26197338e-01 8.33152771e-01 -5.19879639e-01\n", + " -5.87162495e-01 7.00028777e-01 5.87545931e-01 3.44333351e-01\n", + " -1.21452332e+00 -1.60075836e-02 -4.13830161e-01 7.08380222e-01\n", + " 6.98686779e-01 -3.71451080e-01 7.19541430e-01 -1.31629243e-01\n", + " -3.51779093e-03 -4.16871399e-01 2.86455989e-01 -4.04782057e-01\n", + " 6.48040712e-01 1.16212451e+00 -4.72960502e-01 4.89309549e-01\n", + " 2.65322685e-01 1.08556998e+00 3.76119316e-01 -9.22443628e-01\n", + " -2.30535269e-01 3.80907744e-01 -1.21657550e-01 -7.74114653e-02\n", + " -4.94731247e-01 -2.95164287e-01 -7.44602159e-02 -3.36528540e-01\n", + " -1.29304945e+00 6.02036677e-02 1.09093451e+00 -3.91466200e-01\n", + " 5.79024076e-01 2.17555508e-01 -2.87612617e-01 1.94201842e-01\n", + " -5.74356079e-01 6.11653149e-01 3.91429722e-01 -3.38845611e-01\n", + " -5.18676221e-01 3.06476057e-01 -6.03954494e-01 1.80913270e-01\n", + " 3.46696883e-01 -7.77428970e-02 9.09019187e-02 7.00113922e-02\n", + " -8.24879780e-02 9.02246058e-01 -4.05861020e-01 -1.73239093e-02\n", + " 9.99144390e-02 6.32875934e-02 1.64506719e-01 7.46182650e-02\n", + " 8.13847661e-01 6.35026872e-01 -5.65356433e-01 2.60190815e-01\n", + " 5.89712597e-02 1.32967487e-01 -3.05918992e-01 -4.84102726e-01\n", + " -2.93348193e-01 9.84516323e-01 -5.84636986e-01 -1.80695757e-01\n", + " -4.88535255e-01 -2.06880197e-01 5.56958199e-01 -5.00714779e-01\n", + " -3.74321729e-01 1.23296812e-01 -1.32563877e+00 1.42004490e-01\n", + " -3.09263110e-01 8.21540058e-01 -3.08676720e-01 -2.71081656e-01\n", + " -1.69072881e-01 -7.17817128e-01 1.15952206e+00 -2.35318258e-01\n", + " -2.08242297e-01 -3.95812124e-01 -2.92442855e-04 -6.71797633e-01\n", + " 7.18716919e-01 -2.33397886e-01 3.41659814e-01 9.42737758e-01\n", + " 1.77107930e-01 3.04257780e-01 -8.72963190e-01 -2.08007488e-02\n", + " -4.36730206e-01 -8.52538705e-01 6.77212477e-01 4.66120034e-01\n", + " -1.25073135e+00 -5.81974089e-01 1.02274671e-01 -2.64533013e-01\n", + " -2.30871942e-02 7.05368400e-01 4.40623671e-01 -2.83513665e-01\n", + " 1.27129152e-01 4.41659719e-01 -3.14408034e-01 1.47856116e-01\n", + " -1.13659382e+00 -2.17726022e-01 6.23159945e-01 1.18012480e-01\n", + " -1.65116265e-01 -6.40773118e-01 5.46495199e-01 4.17034745e-01\n", + " -1.93105310e-01 5.52878752e-02 1.43081635e-01 -6.16967618e-01\n", + " 4.15407091e-01 1.42037952e+00 3.80081654e-01 9.99245882e-01\n", + " -3.42559338e-01 6.73619032e-01 -2.07689494e-01 6.35467231e-01\n", + " 1.48779273e+00 -2.75697887e-01 -2.63629168e-01 -7.31748194e-02\n", + " 1.89586950e-03 -7.69075811e-01 -4.77384597e-01 1.02100265e+00\n", + " -4.82297450e-01 -1.20669641e-01 4.15332645e-01 5.91618530e-02\n", + " 1.03797054e+00 -5.47217965e-01 3.13258022e-01 6.81133032e-01\n", + " -3.68556559e-01 5.69961190e-01 -3.00340116e-01 -4.80755568e-01\n", + " -5.02778068e-02 6.17770433e-01 1.89927742e-02 -3.77490193e-01\n", + " 1.41768321e-01 7.80981660e-01 -3.84240091e-01 5.52276969e-02\n", + " -9.57285464e-02 -7.23627508e-01 1.49730116e-01 1.09861791e+00\n", + " -6.91233218e-01 -5.31100594e-02 1.20074987e-01 -1.44811803e-02\n", + " -4.69043225e-01 -4.62422192e-01 2.01178610e-01 -8.88655260e-02\n", + " -4.60181028e-01 -3.73510808e-01 -2.60684043e-02 -1.17209144e-01\n", + " 9.15948953e-03 4.16712940e-01 2.67710060e-01 5.27855575e-01\n", + " -4.40576792e-01 -3.54045630e-01 -5.88242970e-02 1.88382208e-01\n", + " 8.21217671e-02 -6.81433916e-01 -3.07059195e-02 5.67822754e-01\n", + " 5.29748499e-01 -4.99385774e-01 -3.09617907e-01 -1.02642417e-01\n", + " -6.61773622e-01 4.33030814e-01 5.99925756e-01 -6.76590979e-01\n", + " 5.00318930e-02 -5.10165215e-01 7.59146929e-01 -2.47493740e-02\n", + " -1.80142477e-01 5.19516230e-01 1.07324533e-01 -5.00668824e-01\n", + " 3.53607774e-01 -1.62214652e-01 -1.60504803e-01 4.40871775e-01\n", + " -1.78438649e-01 -5.05139232e-01 -1.01235890e+00 2.48117894e-01\n", + " -5.11188447e-01 1.74770236e-01 8.29101562e-01 -1.00228179e+00\n", + " 3.27334374e-01 -2.19933197e-01 -1.23447709e-01 -1.61241710e-01\n", + " -6.17850162e-02 1.88073561e-01 -1.02920800e-01 1.72364756e-01\n", + " -5.08176625e-01 5.49406707e-01 4.36621845e-01 -9.10659850e-01\n", + " 1.41511306e-01 -6.20780662e-02 5.00342309e-01 -5.74239850e-01\n", + " -4.36579466e-01 -7.88330659e-02 9.81152296e-01 -6.37921989e-01\n", + " 1.56502441e-01 5.28143011e-02 -2.14096397e-01 1.34172678e-01\n", + " 3.52504551e-01 2.93374956e-01 1.28061175e-01 -6.71271324e-01\n", + " -5.17482087e-02 9.91789341e-01 2.31303737e-01 -4.96764183e-02\n", + " -4.83383715e-01 1.00241154e-01 -9.16052520e-01 -4.53378320e-01\n", + " -2.59417981e-01 2.32401658e-02 8.89824569e-01 4.41391945e-01\n", + " -1.56938568e-01 3.17317069e-01 -1.41847029e-01 -4.74133998e-01\n", + " 5.93121052e-01 -4.54644859e-01 -1.88611615e-02 -2.20847756e-01\n", + " 1.27704597e+00 1.14258385e+00 -2.08520472e-01 -5.56985915e-01\n", + " -4.85482454e-01 1.03180850e+00 3.02320093e-01 8.62334788e-01\n", + " -4.52501655e-01 -7.09908187e-01 2.07147360e-01 8.27097178e-01\n", + " -1.34769559e-01 2.70543844e-02 1.14952639e-01 1.12328768e-01\n", + " -1.82753161e-01 -5.83577991e-01 -2.67232358e-01 3.06137502e-01\n", + " -3.60244513e-01 -4.89607424e-01 4.26346302e-01 -2.05353618e-01\n", + " -1.78375185e-01 -5.30243039e-01 3.66937250e-01 -2.27350473e-01\n", + " 5.51085889e-01 -9.55034792e-02 5.75911224e-01 -3.79808754e-01\n", + " 4.69766632e-02 -8.22235718e-02 4.66322541e-01 -1.28976583e+00\n", + " 1.69037819e-01 -1.40805855e-01 5.04209340e-01 -3.14520717e-01\n", + " 6.52220368e-01 -4.91605103e-02 4.92532998e-01 5.54282486e-01\n", + " -7.60390162e-01 -4.49835509e-02 -3.98927361e-01 -8.29735920e-02\n", + " 9.91931617e-01 1.01718232e-02 3.10495883e-01 -6.12422764e-01\n", + " -6.91284657e-01 2.50425875e-01 5.26148438e-01 1.75509036e-01\n", + " -5.44773042e-01 -4.83786687e-02 -3.26678038e-01 -2.47601882e-01\n", + " -3.10142428e-01 -2.54360795e-01 -4.59155262e-01 2.93214530e-01\n", + " -8.43664169e-01 -3.16741914e-01 -7.90408969e-01 2.47179002e-01\n", + " 1.41189858e-01 2.63226360e-01 -2.08415493e-01 4.52487826e-01\n", + " -7.96849310e-01 5.66900849e-01 3.07065308e-01 1.90365687e-02\n", + " -1.06058681e+00 -2.79941678e-01 -5.16691566e-01 5.78798838e-02\n", + " 8.69804263e-01 1.23412299e+00 -1.03490859e-01 1.25427917e-01\n", + " 6.19832054e-02 -3.40872526e-01 -7.17463791e-02 1.02375376e+00\n", + " 9.17611457e-03 9.19028342e-01 -9.90511596e-01 -1.21138118e-01\n", + " 3.49915445e-01 -2.76705950e-01 -2.73045301e-01 -1.71323568e-01\n", + " -7.79036582e-01 4.91064861e-02 3.88968408e-01 -5.20953476e-01\n", + " 4.72675890e-01 6.44392729e-01 -5.94647646e-01 1.18426876e-02\n", + " -2.66862899e-01 -2.69197404e-01 -2.47342438e-02 -4.46460307e-01\n", + " -3.29050332e-01 1.10608347e-01 1.80856302e-01 4.18497533e-01\n", + " -8.57788861e-01 -1.11232686e+00 -3.16156477e-01 1.11513436e+00\n", + " -3.31797525e-02 -3.43456089e-01 -2.36101791e-01 -3.57465059e-01\n", + " 6.97787225e-01 3.23261678e-01 -4.75672901e-01 1.92753017e-01\n", + " 5.07825613e-01 5.27096868e-01 4.57796216e-01 -2.24204376e-01\n", + " -3.16471010e-01 -5.49207807e-01 -3.38898331e-01 -2.47189865e-01\n", + " -1.38754055e-01 6.57365263e-01 -2.72738021e-02 1.05384871e-01\n", + " -4.50470716e-01 -2.81259924e-01 7.15809882e-01 -2.25987941e-01\n", + " 3.22018743e-01 1.32866964e-01 1.36634529e-01 -3.27936232e-01\n", + " 8.31288278e-01 2.92455144e-02 -5.97056866e-01 -2.27611259e-01\n", + " -2.67553270e-01 2.70353675e-01 -1.04598713e+00 -1.91104680e-01\n", + " 1.35223448e+00 -8.94531131e-01 -8.30622241e-02 -8.39449763e-02\n", + " -5.82324743e-01 -6.32739738e-02 2.65714437e-01 1.77459985e-01\n", + " -4.49232817e-01 4.34464663e-02 -3.41450363e-01 4.12399560e-01\n", + " -3.42403084e-01 8.45672190e-01 -1.73872605e-01 1.35635889e+00\n", + " -7.05278516e-01 1.33494520e+00 -6.51135027e-01 -1.17176580e+00\n", + " 6.32129014e-01 -7.74976850e-01 1.23685308e-01 -4.68495637e-01\n", + " -7.60245919e-02 4.23329443e-01 2.31129721e-01 1.27742863e+00\n", + " -7.44583130e-01 2.28224501e-01 -8.70347738e-01 2.83463627e-01\n", + " -3.56241339e-03 -5.49434841e-01 -4.58050072e-01 1.15442646e+00\n", + " 2.40567893e-01 -1.97578698e-01 2.91451395e-01 8.13631237e-01\n", + " 3.86673540e-01 6.56302869e-02 -8.87026250e-01 4.85067457e-01\n", + " -3.91458631e-01 1.23444192e-01 -2.05788687e-01 1.73837498e-01\n", + " 1.53300762e-01 -4.39879447e-01 6.45275831e-01 -3.48697603e-01\n", + " -5.96876800e-01 7.97479227e-02 -5.32552361e-01 -1.23494053e+00\n", + " 1.95328936e-01 -1.36018276e+00 -3.26980174e-01 -6.96005106e-01\n", + " -2.04857096e-01 -5.17473975e-03 -5.85553765e-01 5.79926908e-01\n", + " 8.44058394e-01 4.85576503e-02 4.51126784e-01 6.05915003e-02\n", + " -7.52648950e-01 7.32060382e-03 -4.16672826e-01 -4.53833491e-01\n", + " -5.78836836e-02 5.14877677e-01 4.25466686e-01 5.64170957e-01\n", + " -9.80011404e-01 8.12970996e-01 -5.29940724e-01 -8.39108467e-01\n", + " 7.16545880e-01 4.95020986e-01 -7.97052324e-01 1.28942788e-01\n", + " -3.02132368e-01 -8.33546221e-02 -4.62123573e-01 -1.43121004e-01\n", + " 1.03349197e+00 4.60390270e-01 -3.30537766e-01 -2.36379713e-01\n", + " -1.97481513e-01 1.48095507e-02 -1.07769620e+00 -1.08410113e-01\n", + " 2.75419831e-01 5.57831042e-02 -2.50686586e-01 -9.27690387e-01\n", + " -4.59981300e-02 -4.01330829e-01 7.14286208e-01 6.12324923e-02\n", + " -8.77659023e-01 5.33353448e-01 -3.26849997e-01 -1.55515403e-01\n", + " 8.83250296e-01 4.38034087e-02 -2.65803058e-02 1.36585847e-01\n", + " -5.74741125e-01 -5.89603901e-01 -2.64363587e-01 -1.14045501e+00\n", + " -8.15228671e-02 -7.95546234e-01 -9.73720908e-01 -7.06167817e-01\n", + " 7.06653714e-01 -3.17953378e-01 -9.40511346e-01 6.16708994e-01\n", + " 3.71810526e-01 7.39454292e-03 1.12828538e-01 -1.28965211e+00\n", + " -7.10735247e-02 1.22390676e+00 -2.59916753e-01 4.34437007e-01\n", + " -5.64912796e-01 -7.83377364e-02 -5.49387753e-01 -1.60182846e+00\n", + " -8.86194944e-01 -1.04885674e+00 -9.94023588e-03 -7.11398542e-01\n", + " -2.63089061e-01 3.24644834e-01 5.88831961e-01 -1.23947762e-01]\n", + "\n" + ] + } + ], + "source": [ + "#for sentence, embedding in zip(df2list, sentence_embeddings):\n", + " print(\"Sentence:\", df2list[0])\n", + " print(\"Embedding:\", sentence_embeddings[0])\n", + " print(\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "ClctPcupxOPi" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import tensorflow as tf\n", + "config = tf.ConfigProto()\n", + "config.gpu_options.allow_growth = True\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "gvSBRxoQCq9H" + }, + "outputs": [], + "source": [ + "\n", + "# save numpy array as npy file\n", + "from numpy import asarray\n", + "from numpy import save\n", + "# define data\n", + "\n", + "# save to npy file\n", + "save('/content/drive/My Drive/full_text_sent_embeddings.csv.npy', sentence_embeddings)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "Oh5izIa7kPqL" + }, + "outputs": [], + "source": [ + "pd2 = model_preds" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "LHsmqEZyGhGu" + }, + "outputs": [], + "source": [ + "gold_std = pd2['gold'].to_list()\n", + "oracle_std = pd2['oracle'].to_list()\n", + "lead_std = pd2['lead'].to_list()\n", + "ptgen_std = pd2['ptgen'].to_list()\n", + "topic_convs2s = pd2['topic_convs2s'].to_list\n", + "\n", + "convs2s_std = pd2['convs2s'].to_list()" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "nI516uo7Gz8J" + }, + "outputs": [], + "source": [ + "pd_convs2s = model.encode(convs2s_std)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 426, + "status": "ok", + "timestamp": 1588694718625, + "user": { + "displayName": "Parthvi Shah", + "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj14forhnYAb8tD0SHMMufGgXmEIpTA6BXAnPZ5Uvg=s64", + "userId": "12920190590086167612" + }, + "user_tz": 240 + }, + "id": "J-NTna2llTPo", + "outputId": "37f37efa-7c32-4cdf-ffcc-04af38523eaf" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "11334" + ] + }, + "execution_count": 30, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "len(pd_convs2s)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "-01FDiFAG7SK" + }, + "outputs": [], + "source": [ + "save('/content/drive/My Drive/conv2s_std_sent_embeddings.csv.npy', pd_convs2s )" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "GGzUrHmmHEVH" + }, + "outputs": [], + "source": [ + "pd_oracle = (model.encode(oracle_std))\n", + "pd_lead = model.encode(lead_std)\n", + "\n", + "save('/content/drive/My Drive/oracle_sent_embeddings.csv.npy', pd_oracle)\n", + "save('/content/drive/My Drive/lead_sent_embeddings.csv.npy', pd_lead)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "FB0ZGXyklfqj" + }, + "outputs": [], + "source": [ + "gold_std = pd2['gold'].to_list()\n", + "ptgen_std = pd2['ptgen'].to_list()\n", + "topic_convs2s = pd2['topic_convs2s'].to_list()\n", + "\n", + "#pd_gold = (model.encode(gold_std))\n", + "#pd_ptgen = model.encode(ptgen_std)\n", + "pd_topic = model.encode(topic_convs2s)\n", + "\n", + "#save('/content/drive/My Drive/gold_sent_embeddings.csv.npy', pd_gold)\n", + "#save('/content/drive/My Drive/ptgen_sent_embeddings.csv.npy', pd_ptgen)\n", + "save('/content/drive/My Drive/topic_sent_embeddings.csv.npy', pd_topic)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 251, + "status": "ok", + "timestamp": 1588696693970, + "user": { + "displayName": "Parthvi Shah", + "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj14forhnYAb8tD0SHMMufGgXmEIpTA6BXAnPZ5Uvg=s64", + "userId": "12920190590086167612" + }, + "user_tz": 240 + }, + "id": "hiGZgDSHlguI", + "outputId": "51360a3f-85c2-4baf-a122-d879a5c17b4f" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "array([-2.49393746e-01, 3.55545551e-01, 8.40745568e-01, -1.64429605e-01,\n", + " 7.13316858e-01, -8.73796046e-02, 9.96831238e-01, 3.26661736e-01,\n", + " -1.40925929e-01, 6.71937466e-02, 2.52174828e-02, -6.39251322e-02,\n", + " 4.05319780e-01, 9.74939540e-02, -5.47203779e-01, 4.59083021e-01,\n", + " -3.31218541e-01, -2.32927576e-02, -2.52150387e-01, -1.80637911e-01,\n", + " 1.14652000e-01, -1.07270384e+00, 8.69179904e-01, 3.60032111e-01,\n", + " 1.10179150e+00, -8.93468186e-02, -2.61973664e-02, 1.55868441e-01,\n", + " -1.14413011e+00, 3.08493018e-01, -7.01565266e-01, 1.32840022e-01,\n", + " -5.67640603e-01, 9.48267281e-02, 2.52067268e-01, 1.04207027e+00,\n", + " -4.48491991e-01, -5.61940134e-01, -4.18673605e-01, -5.51107386e-03,\n", + " -3.10015261e-01, 2.72015125e-01, 4.68301699e-02, 3.01400840e-01,\n", + " -2.83310711e-01, 3.89204383e-01, 2.65798837e-01, 7.60998905e-01,\n", + " 1.30835462e+00, -9.55899417e-01, -4.49460685e-01, 4.31870371e-01,\n", + " 1.38546979e+00, 2.01663896e-01, -4.43949789e-01, -6.65321529e-01,\n", + " 5.82476594e-02, -1.08461273e+00, -7.57258475e-01, -4.47398007e-01,\n", + " -6.51550472e-01, 6.21622145e-01, -9.39760506e-01, -6.78224206e-01,\n", + " -1.50421292e-01, -7.54414380e-01, -2.47485802e-01, 2.88093418e-01,\n", + " -1.43169534e+00, -1.05408919e+00, -4.54685777e-01, -1.06542194e+00,\n", + " -9.18708444e-01, -8.33249331e-01, 5.51660001e-01, -1.90148130e-02,\n", + " -4.14968887e-03, -2.03300244e-03, 1.54487538e+00, 3.93661916e-01,\n", + " 4.98047829e-01, -3.28976274e-01, 4.57354903e-01, -4.32964265e-01,\n", + " 8.23719740e-01, -6.63275793e-02, -3.62686574e-01, -1.26276302e+00,\n", + " -1.02167316e-01, 2.10337549e-01, -5.57896271e-02, -1.18431173e-01,\n", + " 2.79050499e-01, 2.89368212e-01, -1.44523036e+00, -1.75305113e-01,\n", + " -3.04808933e-02, -2.25151628e-01, 9.22945142e-02, 3.41170847e-01,\n", + " 5.15776455e-01, -1.00185331e-02, -2.28097662e-01, 8.89256597e-01,\n", + " -2.87078768e-01, 4.26918179e-01, -3.49587470e-01, -2.23769367e-01,\n", + " 3.36484969e-01, 1.22584254e-01, -1.39823034e-01, 1.94119155e-01,\n", + " -1.44698933e-01, -1.30105987e-01, 1.10270754e-01, -5.94217002e-01,\n", + " -4.41031128e-01, 5.98483622e-01, 1.30798845e-02, 7.76103675e-01,\n", + " 1.31243503e+00, -7.07272172e-01, 5.27194375e-03, 3.70647758e-01,\n", + " 1.63847700e-01, 7.09637821e-01, -1.78017423e-01, 4.87519115e-01,\n", + " -1.11050045e+00, -2.88480490e-01, -3.70289385e-01, 5.39557338e-01,\n", + " -9.24946666e-01, -3.72751147e-01, -3.02462012e-01, -7.69096091e-02,\n", + " -4.69963551e-02, -9.83790696e-01, 8.16860974e-01, -1.51324198e-01,\n", + " -1.90966159e-01, 1.80544928e-02, 3.93441468e-01, -2.24056423e-01,\n", + " 1.08366527e-01, -2.52915114e-01, 1.73744768e-01, -6.13069475e-01,\n", + " -1.65123358e-01, -4.07167990e-03, 3.68991584e-01, -4.56986547e-01,\n", + " -4.16361600e-01, 8.92402411e-01, -5.21044552e-01, -1.87277392e-01,\n", + " -1.62473157e-01, -6.45687699e-01, -6.15987957e-01, 2.73718268e-01,\n", + " 6.22510195e-01, 1.56594425e-01, 2.03701153e-01, 1.20552826e+00,\n", + " 4.80544746e-01, 4.58097696e-01, 1.05176711e+00, -7.53293812e-01,\n", + " 1.03141713e+00, -3.85875165e-01, -5.46510577e-01, 8.96388590e-01,\n", + " 2.37389159e+00, -3.76502424e-01, 9.47063088e-01, -6.44237623e-02,\n", + " -6.47413194e-01, -7.25741863e-01, -4.92292762e-01, 3.71965542e-02,\n", + " -4.55070376e-01, 4.87868041e-01, -4.78097290e-01, 1.30191886e+00,\n", + " -2.28794850e-02, -9.63260755e-02, 8.26177597e-01, -3.46503615e-01,\n", + " -4.75201577e-01, -1.68572411e-01, -5.93781173e-01, -3.92955780e-01,\n", + " 1.28475976e+00, -2.20944792e-01, -3.11465055e-01, -2.57965744e-01,\n", + " 3.55479419e-01, -1.29214287e-01, -5.15759826e-01, 8.46833825e-01,\n", + " -1.53319642e-01, 3.41585100e-01, -2.01389045e-01, -1.44663304e-01,\n", + " 3.99438918e-01, 6.13014519e-01, 3.21779907e-01, -1.52030066e-01,\n", + " -7.70192742e-01, -1.63677543e-01, -7.13113129e-01, 9.06727612e-02,\n", + " 9.71679211e-01, 6.59118533e-01, 1.61382926e+00, -8.87143314e-02,\n", + " -5.13848782e-01, -3.19341600e-01, -3.83573472e-01, 9.54670370e-01,\n", + " -4.38398629e-01, 4.13913041e-01, -1.80506393e-01, -4.72116098e-02,\n", + " -2.15130150e-01, 2.52128690e-01, -3.55281532e-01, -6.87680602e-01,\n", + " -3.88511389e-01, 3.16014379e-01, -6.50137290e-02, 5.92716634e-01,\n", + " -1.27615976e+00, 3.68520856e-01, -2.28888601e-01, -8.51670980e-01,\n", + " -2.04416203e+00, 2.32175335e-01, 8.16580534e-01, -1.24359989e+00,\n", + " -5.88607453e-02, 4.46402252e-01, 2.14994311e-01, 7.32027173e-01,\n", + " -2.65485436e-01, -1.20734558e-01, 9.45809722e-01, -3.42456698e-01,\n", + " 2.02312395e-01, 5.79628758e-02, -4.19429541e-01, -4.11847055e-01,\n", + " 3.45418543e-01, -3.75417918e-01, -5.66534579e-01, -4.17458028e-01,\n", + " 2.44206294e-01, 6.57878816e-01, 2.94536978e-01, 2.65010744e-01,\n", + " 1.10652938e-01, 2.38814101e-01, 9.65339363e-01, 3.27548623e-01,\n", + " -2.95624375e-01, 3.89568321e-02, -4.12921518e-01, 1.17366917e-01,\n", + " -7.42350817e-01, -2.36460283e-01, -2.86962628e-01, -1.26362514e+00,\n", + " 9.99711379e-02, 7.98829257e-01, -8.37211072e-01, -1.91421196e-01,\n", + " -1.09623373e+00, 1.10254558e-02, 4.96115416e-01, 4.09650415e-01,\n", + " 2.63290077e-01, 9.69447374e-01, 1.71709478e-01, -4.26970065e-01,\n", + " -1.44290698e+00, 8.98643494e-01, -8.21068227e-01, 3.22436750e-01,\n", + " -1.92158177e-01, -4.31864530e-01, 5.35919189e-01, -1.21521674e-01,\n", + " 4.85530257e-01, 9.64839906e-02, -1.39240041e-01, -6.56655192e-01,\n", + " 3.52074236e-01, -2.12064654e-01, 7.21648812e-01, 7.64570236e-01,\n", + " -1.85587406e-01, 7.38687158e-01, -1.07719266e+00, -2.31620878e-01,\n", + " -9.04033124e-01, -3.70893836e-01, -3.28874350e-01, 5.29251516e-01,\n", + " 5.95507085e-01, 1.30225435e-01, -6.01871967e-01, -7.26839662e-01,\n", + " 1.47359204e-02, 1.47545540e+00, 8.89362156e-01, 1.82583351e-02,\n", + " 8.17749351e-02, -8.84984136e-01, -3.22451234e-01, -6.28430426e-01,\n", + " -7.58013070e-01, 9.90864858e-02, 3.28277767e-01, 1.46734670e-01,\n", + " 8.42285693e-01, -1.31467950e+00, 6.45728707e-01, 5.51947534e-01,\n", + " 1.85308736e-02, -2.39609301e-01, -3.62775624e-01, -7.55185366e-01,\n", + " 6.79201603e-01, 7.79427230e-01, 5.81069827e-01, 4.52591926e-01,\n", + " -4.16156143e-01, 2.16602795e-02, 3.73198837e-01, 6.84149265e-01,\n", + " 1.74464881e+00, -1.11692357e+00, -2.52093971e-01, 5.32133698e-01,\n", + " 5.34290671e-01, -1.53796420e-01, -8.20366815e-02, 9.99696970e-01,\n", + " -8.29011500e-01, -3.88830572e-01, 3.93109590e-01, 5.53316414e-01,\n", + " 1.98532796e+00, 1.05684742e-01, 4.50618953e-01, 1.96191519e-01,\n", + " -9.24107015e-01, 4.36459184e-01, 8.57997388e-02, -4.89233285e-01,\n", + " -4.28020537e-01, 1.06130369e-01, 7.75166377e-02, 1.36477351e-01,\n", + " -5.51245883e-02, 2.81391561e-01, -2.17253268e-01, -1.44277528e-01,\n", + " 1.47262469e-01, -2.18754649e-01, 7.17564642e-01, 4.24703956e-01,\n", + " -4.85784352e-01, -6.38500392e-01, 6.44760191e-01, -8.57595444e-01,\n", + " -4.66687858e-01, -6.58505410e-02, -4.00293283e-02, -3.87632012e-01,\n", + " -2.19157472e-01, -4.99850720e-01, 5.88204861e-01, 2.27562889e-01,\n", + " 2.64165878e-01, -4.82326746e-01, 5.08191586e-01, -6.05781555e-01,\n", + " 5.36609054e-01, 1.22692071e-01, 8.51909161e-01, 8.30551863e-01,\n", + " -2.84851372e-01, 2.23222505e-02, 2.01575205e-01, -3.43622565e-01,\n", + " -3.31527665e-02, -1.42384648e-01, -2.32202490e-03, 3.55188310e-01,\n", + " -3.78149748e-01, 8.83153453e-02, 7.67562747e-01, -6.87483102e-02,\n", + " 1.19170964e+00, -1.01780556e-01, 1.97363049e-01, -2.14659497e-01,\n", + " -3.05746235e-02, -4.60766256e-02, 2.23918363e-01, -3.91507924e-01,\n", + " -3.66992891e-01, -3.05633605e-01, -2.86303103e-01, 5.33006907e-01,\n", + " -4.18333918e-01, -6.22102499e-01, -1.04202700e+00, 1.03934741e+00,\n", + " 4.41300333e-01, -9.16612804e-01, 5.77206500e-02, -5.71922243e-01,\n", + " 7.84756899e-01, 1.20510173e+00, 5.62511571e-03, -5.33089221e-01,\n", + " -8.19473326e-01, -4.29700576e-02, 2.16078069e-02, -2.79201150e-01,\n", + " -4.57563847e-01, -3.03842485e-01, 3.37618381e-01, -1.22498786e+00,\n", + " 1.04691945e-01, 1.23961210e-01, 3.35413888e-02, -8.46883595e-01,\n", + " -9.28134620e-01, -6.19968832e-01, 3.22971046e-01, -6.72498763e-01,\n", + " -7.03492343e-01, 1.82426840e-01, -2.50913024e-01, -7.66925747e-04,\n", + " 1.10183634e-01, -9.65966564e-03, 6.33796677e-02, -6.56146586e-01,\n", + " 5.45270443e-01, 7.81764627e-01, 6.39190733e-01, 2.57739097e-01,\n", + " -9.38177407e-01, 3.30873609e-01, -1.58334231e+00, 1.23733625e-01,\n", + " 3.33918601e-01, 4.11609150e-02, 1.65941760e-01, 5.69163680e-01,\n", + " 1.16662753e+00, -1.49972484e-01, 7.47852027e-01, -8.02681625e-01,\n", + " 1.12073675e-01, -2.42425397e-01, 1.23742074e-01, 2.41894752e-01,\n", + " 2.40804762e-01, 8.49889100e-01, -1.50691777e-01, -1.33513355e+00,\n", + " 9.96859297e-02, 8.35739672e-01, -4.01957452e-01, 7.68898904e-01,\n", + " -6.67630613e-01, -4.10893679e-01, 4.89587367e-01, 5.88700533e-01,\n", + " -7.13301718e-01, -3.47290456e-01, -5.58822811e-01, 3.67456019e-01,\n", + " 2.97350734e-01, 1.94791127e-02, 6.25822246e-01, 4.80570406e-01,\n", + " -3.47648770e-01, -1.21988155e-01, 7.27541864e-01, -6.13131344e-01,\n", + " 2.25414246e-01, 3.64444107e-01, 2.66586483e-01, 9.30866376e-02,\n", + " 5.82637668e-01, -4.68089253e-01, 4.66064394e-01, 4.08582389e-02,\n", + " 1.21557176e+00, 3.93667310e-01, -6.77401721e-01, -4.70113009e-01,\n", + " 4.92161751e-01, -3.90507013e-01, 4.27331954e-01, 3.32238764e-01,\n", + " 3.38857234e-01, 4.76251930e-01, 1.97377875e-01, -2.57953584e-01,\n", + " -4.11690950e-01, -1.66446075e-01, 2.56902009e-01, -3.19622308e-01,\n", + " 4.97849733e-01, -6.64556772e-02, 6.34490728e-01, -4.19379205e-01,\n", + " 1.75736934e-01, -1.28936032e-02, -4.80829775e-01, 7.29133725e-01,\n", + " -1.15373170e+00, -4.68214095e-01, -2.51314104e-01, -5.60556054e-01,\n", + " 2.05903053e-01, 1.02340698e+00, -8.12693775e-01, 8.91640186e-02,\n", + " -3.58389616e-01, -4.25394565e-01, -4.94031936e-01, -1.68022409e-01,\n", + " -1.86451897e-01, 1.76694974e-01, 1.38197792e+00, -4.18610796e-02,\n", + " -1.43029845e+00, -5.87282300e-01, 1.38471007e-01, 2.06522599e-01,\n", + " -7.01054931e-01, -5.91700077e-01, 2.19658256e-01, -5.19029610e-02,\n", + " 2.41869271e-01, 9.06537622e-02, 6.16174415e-02, 2.05144539e-01,\n", + " 1.33529961e-01, -3.07721466e-01, 3.61509323e-01, 1.05525863e+00,\n", + " 1.75540876e-02, 5.09740293e-01, -5.61913908e-01, -5.95469952e-01,\n", + " -1.14460416e-01, -5.19804396e-02, -3.35403979e-01, -7.20580518e-01,\n", + " -3.73544514e-01, -8.22639585e-01, 1.22016883e+00, -1.62949041e-01,\n", + " 9.83711898e-01, 2.50142783e-01, -3.53305191e-02, -3.52598071e-01,\n", + " -3.57206404e-01, -1.28114030e-01, 4.59522337e-01, 4.64756817e-01,\n", + " 1.38224876e+00, -4.86620575e-01, -1.13374686e+00, -1.63106009e-01,\n", + " -4.18560028e-01, -5.52601099e-01, -1.01045199e-01, 4.96463120e-01,\n", + " -1.98574215e-01, -1.07956804e-01, -6.17922992e-02, -1.60778150e-01,\n", + " 9.39488113e-01, 2.72703677e-01, 1.17156494e+00, 2.76189774e-01,\n", + " -3.50095987e-01, -4.59600478e-01, 3.96867007e-01, 2.01868162e-01,\n", + " -4.67718810e-01, -9.29773226e-03, -3.18743318e-01, 3.22680354e-01,\n", + " -4.37800109e-01, 4.71924126e-01, 3.36840600e-01, 2.39422306e-01,\n", + " -1.18501353e+00, -1.84440911e-01, 2.82456279e-01, -4.63252038e-01,\n", + " -3.82574141e-01, -1.18009426e-01, -4.99110311e-01, -4.44564015e-01,\n", + " 7.35277772e-01, -4.01032388e-01, -1.05574834e+00, 2.81449147e-02,\n", + " -1.20200098e-01, 3.68629873e-01, 1.50149077e-01, 3.99740875e-01,\n", + " 1.50006518e-01, -7.24556983e-01, 2.01607674e-01, -2.51645744e-01,\n", + " -5.79651713e-01, -1.51620016e-01, -5.70571050e-02, -3.77611995e-01,\n", + " -1.26545906e-01, 4.89236534e-01, -4.93703008e-01, 3.93116206e-01,\n", + " -3.28222811e-01, 5.56846440e-01, -6.20215356e-01, 2.67083272e-02,\n", + " 2.19975427e-01, 1.68344796e+00, -6.40632331e-01, -1.91396087e-01,\n", + " 5.96461236e-01, -1.43628848e+00, -7.13006184e-02, 1.46202624e-01,\n", + " 6.16192698e-01, 8.03218722e-01, -8.72258469e-02, 9.47354734e-01,\n", + " -5.71915694e-02, -3.42574686e-01, -1.34072289e-01, 1.41318530e-01,\n", + " -9.15568173e-02, -1.06428392e-01, -4.70672011e-01, 9.49284852e-01,\n", + " 1.56982198e-01, -3.59272957e-01, -8.18370208e-02, 4.38900977e-01,\n", + " 3.33911628e-01, 4.39999968e-01, -2.50947960e-02, 1.69420630e-01,\n", + " -3.62932742e-01, 1.86050683e-01, -2.32035354e-01, -4.56443250e-01,\n", + " 4.42105941e-02, -3.42027813e-01, 5.90060949e-01, -3.46963942e-01,\n", + " -4.55815792e-02, 1.22717954e-01, 1.15529525e+00, -6.79676771e-01,\n", + " -5.50501525e-01, -7.31996894e-01, -3.49435210e-01, -7.77301788e-01,\n", + " 7.12823331e-01, -2.60727536e-02, -6.46154642e-01, 1.84764698e-01,\n", + " -9.05477941e-01, 5.38578331e-01, 3.88161391e-01, 2.61161476e-01,\n", + " -7.84786522e-01, -7.65537858e-01, -3.70967269e-01, -9.30567905e-02,\n", + " 3.43415529e-01, 6.02382123e-01, 7.32366025e-01, 1.51043504e-01,\n", + " -1.11586559e+00, 2.03587487e-01, -8.96689653e-01, -9.64142919e-01,\n", + " 6.15906537e-01, -4.46668506e-01, -3.37962061e-01, 1.08308327e+00,\n", + " 5.05343139e-01, -5.62603474e-01, 3.15761417e-01, 1.44974661e+00,\n", + " -3.26687515e-01, 1.06681430e+00, 2.39189882e-02, -4.82405350e-02,\n", + " 6.97071627e-02, -2.63475806e-01, -6.80335879e-01, 1.52749375e-01,\n", + " 1.82322115e-01, -2.71387249e-01, 4.83125806e-01, -1.47660136e+00,\n", + " -1.04903914e-01, -4.74589348e-01, -8.27400506e-01, -7.23818168e-02,\n", + " -2.17497274e-01, -4.54875290e-01, -1.91132143e-01, -4.33538586e-01,\n", + " -7.28748739e-02, -2.13675141e-01, -2.07408905e-01, 1.33913949e-01,\n", + " 4.59685475e-01, -7.19693780e-01, 1.98231086e-01, -1.67743266e+00,\n", + " -7.62504153e-03, 2.87113339e-02, -4.45023805e-01, -5.92536449e-01,\n", + " 4.24703002e-01, 7.03995049e-01, 1.53156430e-01, 1.75877601e-01,\n", + " -6.41273737e-01, -2.50131249e-01, -3.59100252e-01, 1.20698377e-01,\n", + " -5.27028032e-02, 8.16464067e-01, -4.03613448e-01, 5.56782424e-01,\n", + " -2.18288735e-01, -6.95395648e-01, -4.18814689e-01, -6.65259242e-01,\n", + " -6.92896068e-01, -1.18170232e-01, 5.01315415e-01, -1.79181278e+00,\n", + " -2.15827659e-01, -4.10065293e-01, -3.92747134e-01, 1.96002111e-01],\n", + " dtype=float32)" + ] + }, + "execution_count": 43, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "pd_gold[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "pIfoLxoEs22c" + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "TPU", + "colab": { + "collapsed_sections": [], + "machine_shape": "hm", + "name": "Sentence_Embedding.ipynb", + "provenance": [ + { + "file_id": "13jIcfZ81B2sUrmtkU2l8hFkNvXwpKzzv", + "timestamp": 1588684750953 + }, + { + "file_id": "https://gist.github.com/ashutoshsingh0223/d6d673a942dd15546fc28e9fce875b51#file-embedding-as-service-ipynb", + "timestamp": 1587330263068 + } + ] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/XLNet_Embedding.ipynb b/XLNet_Embedding.ipynb deleted file mode 100644 index 15d4ec2..0000000 --- a/XLNet_Embedding.ipynb +++ /dev/null @@ -1 +0,0 @@ -{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"XLNet_Embedding.ipynb","provenance":[{"file_id":"https://gist.github.com/ashutoshsingh0223/d6d673a942dd15546fc28e9fce875b51#file-embedding-as-service-ipynb","timestamp":1587330263068}],"collapsed_sections":[],"machine_shape":"hm"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"TPU"},"cells":[{"cell_type":"code","metadata":{"id":"UxJGubvI1qOd","colab_type":"code","outputId":"6f1c4e66-b84a-406b-cd55-00ee60a3e558","executionInfo":{"status":"ok","timestamp":1587355965860,"user_tz":240,"elapsed":1990,"user":{"displayName":"Praxal Suresh Patel","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjZDJ75FrNT9-0GPWBNlo9njpv7eqdcVnrdgzHY=s64","userId":"09765785019516469778"}},"colab":{"base_uri":"https://localhost:8080/","height":136}},"source":["!git clone https://github.com/amansrivastava17/embedding-as-service.git"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Cloning into 'embedding-as-service'...\n","remote: Enumerating objects: 136, done.\u001b[K\n","remote: Counting objects: 100% (136/136), done.\u001b[K\n","remote: Compressing objects: 100% (93/93), done.\u001b[K\n","remote: Total 1983 (delta 52), reused 94 (delta 29), pack-reused 1847\u001b[K\n","Receiving objects: 100% (1983/1983), 1.95 MiB | 13.84 MiB/s, done.\n","Resolving deltas: 100% (1091/1091), done.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"z1uIioWb1sua","colab_type":"code","outputId":"75a5076d-8413-4796-947f-8a49c921b517","executionInfo":{"status":"ok","timestamp":1587355965861,"user_tz":240,"elapsed":1981,"user":{"displayName":"Praxal Suresh Patel","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjZDJ75FrNT9-0GPWBNlo9njpv7eqdcVnrdgzHY=s64","userId":"09765785019516469778"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["cd embedding-as-service/"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/content/embedding-as-service\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"J1eMYEKn22Ib","colab_type":"code","outputId":"3c739691-4927-4245-8b73-d41230f80c0d","executionInfo":{"status":"ok","timestamp":1587355966941,"user_tz":240,"elapsed":3055,"user":{"displayName":"Praxal Suresh Patel","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjZDJ75FrNT9-0GPWBNlo9njpv7eqdcVnrdgzHY=s64","userId":"09765785019516469778"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["!git checkout master\n","# !git pull origin added-xlnet"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Already on 'master'\n","Your branch is up to date with 'origin/master'.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"SRH_Ei3hEdEG","colab_type":"code","outputId":"0e77dbe4-03c4-48c6-ed1f-90f5106c4d9d","executionInfo":{"status":"ok","timestamp":1587355967523,"user_tz":240,"elapsed":3631,"user":{"displayName":"Praxal Suresh Patel","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjZDJ75FrNT9-0GPWBNlo9njpv7eqdcVnrdgzHY=s64","userId":"09765785019516469778"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["!git branch"],"execution_count":0,"outputs":[{"output_type":"stream","text":["* \u001b[32mmaster\u001b[m\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"n6jJkyzCRLIG","colab_type":"code","outputId":"cfb60659-3db1-4113-93d3-5eda7460fd0a","executionInfo":{"status":"ok","timestamp":1587356057985,"user_tz":240,"elapsed":94087,"user":{"displayName":"Praxal Suresh Patel","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjZDJ75FrNT9-0GPWBNlo9njpv7eqdcVnrdgzHY=s64","userId":"09765785019516469778"}},"colab":{"base_uri":"https://localhost:8080/","height":1000}},"source":["!pip install embedding-as-service\n","\n"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Collecting embedding-as-service\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/6f/1e/e670805eda7dde0eb08707f04d074bc0b0d69d1b9fcb6a31dc3fa552c600/embedding_as_service-2.0.0-py3-none-any.whl (140kB)\n","\r\u001b[K |██▍ | 10kB 19.9MB/s eta 0:00:01\r\u001b[K |████▊ | 20kB 2.2MB/s eta 0:00:01\r\u001b[K |███████ | 30kB 2.9MB/s eta 0:00:01\r\u001b[K |█████████▍ | 40kB 2.1MB/s eta 0:00:01\r\u001b[K |███████████▊ | 51kB 2.3MB/s eta 0:00:01\r\u001b[K |██████████████ | 61kB 2.7MB/s eta 0:00:01\r\u001b[K |████████████████▍ | 71kB 3.0MB/s eta 0:00:01\r\u001b[K |██████████████████▊ | 81kB 3.3MB/s eta 0:00:01\r\u001b[K |█████████████████████ | 92kB 3.6MB/s eta 0:00:01\r\u001b[K |███████████████████████▍ | 102kB 3.4MB/s eta 0:00:01\r\u001b[K |█████████████████████████▊ | 112kB 3.4MB/s eta 0:00:01\r\u001b[K |████████████████████████████ | 122kB 3.4MB/s eta 0:00:01\r\u001b[K |██████████████████████████████▍ | 133kB 3.4MB/s eta 0:00:01\r\u001b[K |████████████████████████████████| 143kB 3.4MB/s \n","\u001b[?25hCollecting zmq==0.0.0\n"," Downloading https://files.pythonhosted.org/packages/6e/78/833b2808793c1619835edb1a4e17a023d5d625f4f97ff25ffff986d1f472/zmq-0.0.0.tar.gz\n","Collecting bert-tensorflow==1.0.1\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/a6/66/7eb4e8b6ea35b7cc54c322c816f976167a43019750279a8473d355800a93/bert_tensorflow-1.0.1-py2.py3-none-any.whl (67kB)\n","\u001b[K |████████████████████████████████| 71kB 6.5MB/s \n","\u001b[?25hCollecting numpy==1.16.4\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/87/2d/e4656149cbadd3a8a0369fcd1a9c7d61cc7b87b3903b85389c70c989a696/numpy-1.16.4-cp36-cp36m-manylinux1_x86_64.whl (17.3MB)\n","\u001b[K |████████████████████████████████| 17.3MB 241kB/s \n","\u001b[?25hRequirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from embedding-as-service) (46.1.3)\n","Collecting sentencepiece==0.1.85\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n","\u001b[K |████████████████████████████████| 1.0MB 59.3MB/s \n","\u001b[?25hCollecting tensorflow-hub==0.4.0\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/10/5c/6f3698513cf1cd730a5ea66aec665d213adf9de59b34f362f270e0bd126f/tensorflow_hub-0.4.0-py2.py3-none-any.whl (75kB)\n","\u001b[K |████████████████████████████████| 81kB 11.7MB/s \n","\u001b[?25hCollecting smart-open==1.8.4\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/37/c0/25d19badc495428dec6a4bf7782de617ee0246a9211af75b302a2681dea7/smart_open-1.8.4.tar.gz (63kB)\n","\u001b[K |████████████████████████████████| 71kB 10.6MB/s \n","\u001b[?25hCollecting tensorflow==1.15.0\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/3f/98/5a99af92fb911d7a88a0005ad55005f35b4c1ba8d75fba02df726cd936e6/tensorflow-1.15.0-cp36-cp36m-manylinux2010_x86_64.whl (412.3MB)\n","\u001b[K |████████████████████████████████| 412.3MB 31kB/s \n","\u001b[?25hRequirement already satisfied: requests==2.21.0 in /usr/local/lib/python3.6/dist-packages (from embedding-as-service) (2.21.0)\n","Collecting tqdm==4.32.2\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/9f/3d/7a6b68b631d2ab54975f3a4863f3c4e9b26445353264ef01f465dc9b0208/tqdm-4.32.2-py2.py3-none-any.whl (50kB)\n","\u001b[K |████████████████████████████████| 51kB 7.4MB/s \n","\u001b[?25hCollecting keras==2.2.4\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/5e/10/aa32dad071ce52b5502266b5c659451cfd6ffcbf14e6c8c4f16c0ff5aaab/Keras-2.2.4-py2.py3-none-any.whl (312kB)\n","\u001b[K |████████████████████████████████| 317kB 60.8MB/s \n","\u001b[?25hRequirement already satisfied: pyzmq in /usr/local/lib/python3.6/dist-packages (from zmq==0.0.0->embedding-as-service) (19.0.0)\n","Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from bert-tensorflow==1.0.1->embedding-as-service) (1.12.0)\n","Requirement already satisfied: protobuf>=3.4.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-hub==0.4.0->embedding-as-service) (3.10.0)\n","Requirement already satisfied: boto>=2.32 in /usr/local/lib/python3.6/dist-packages (from smart-open==1.8.4->embedding-as-service) (2.49.0)\n","Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from smart-open==1.8.4->embedding-as-service) (1.12.39)\n","Requirement already satisfied: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0->embedding-as-service) (1.0.8)\n","Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0->embedding-as-service) (0.8.1)\n","Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0->embedding-as-service) (0.2.0)\n","Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0->embedding-as-service) (3.2.0)\n","Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0->embedding-as-service) (0.34.2)\n","Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0->embedding-as-service) (0.9.0)\n","Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0->embedding-as-service) (1.12.1)\n","Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0->embedding-as-service) (1.1.0)\n","Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0->embedding-as-service) (1.28.1)\n","Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0->embedding-as-service) (1.1.0)\n","Collecting tensorflow-estimator==1.15.1\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/de/62/2ee9cd74c9fa2fa450877847ba560b260f5d0fb70ee0595203082dafcc9d/tensorflow_estimator-1.15.1-py2.py3-none-any.whl (503kB)\n","\u001b[K |████████████████████████████████| 512kB 59.0MB/s \n","\u001b[?25hCollecting tensorboard<1.16.0,>=1.15.0\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/1e/e9/d3d747a97f7188f48aa5eda486907f3b345cd409f0a0850468ba867db246/tensorboard-1.15.0-py3-none-any.whl (3.8MB)\n","\u001b[K |████████████████████████████████| 3.8MB 61.1MB/s \n","\u001b[?25hCollecting gast==0.2.2\n"," Downloading https://files.pythonhosted.org/packages/4e/35/11749bf99b2d4e3cceb4d55ca22590b0d7c2c62b9de38ac4a4a7f4687421/gast-0.2.2.tar.gz\n","Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests==2.21.0->embedding-as-service) (3.0.4)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests==2.21.0->embedding-as-service) (2020.4.5.1)\n","Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests==2.21.0->embedding-as-service) (2.8)\n","Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests==2.21.0->embedding-as-service) (1.24.3)\n","Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.6/dist-packages (from keras==2.2.4->embedding-as-service) (1.4.1)\n","Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras==2.2.4->embedding-as-service) (2.10.0)\n","Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from keras==2.2.4->embedding-as-service) (3.13)\n","Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open==1.8.4->embedding-as-service) (0.3.3)\n","Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open==1.8.4->embedding-as-service) (0.9.5)\n","Requirement already satisfied: botocore<1.16.0,>=1.15.39 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open==1.8.4->embedding-as-service) (1.15.39)\n","Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15.0->embedding-as-service) (1.0.1)\n","Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15.0->embedding-as-service) (3.2.1)\n","Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.39->boto3->smart-open==1.8.4->embedding-as-service) (2.8.1)\n","Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.39->boto3->smart-open==1.8.4->embedding-as-service) (0.15.2)\n","Building wheels for collected packages: zmq, smart-open, gast\n"," Building wheel for zmq (setup.py) ... \u001b[?25l\u001b[?25hdone\n"," Created wheel for zmq: filename=zmq-0.0.0-cp36-none-any.whl size=1278 sha256=812817f16aeeccea805a7f1909c4644d032f70ffe28e7dddca24f059967c6c8d\n"," Stored in directory: /root/.cache/pip/wheels/44/7a/7d/ac1d865766b06f9769ac1154bf31dbb5abb3b52ecfe278247a\n"," Building wheel for smart-open (setup.py) ... \u001b[?25l\u001b[?25hdone\n"," Created wheel for smart-open: filename=smart_open-1.8.4-cp36-none-any.whl size=68200 sha256=99b8008476badeba108ed935e2c3a631514f4dfdcaf00245f84301b8bf1273c5\n"," Stored in directory: /root/.cache/pip/wheels/5f/ea/fb/5b1a947b369724063b2617011f1540c44eb00e28c3d2ca8692\n"," Building wheel for gast (setup.py) ... \u001b[?25l\u001b[?25hdone\n"," Created wheel for gast: filename=gast-0.2.2-cp36-none-any.whl size=7540 sha256=9d1d42d3cce6091edff223963fbd1918e13c39e2908bda4c4bce85121b796de4\n"," Stored in directory: /root/.cache/pip/wheels/5c/2e/7e/a1d4d4fcebe6c381f378ce7743a3ced3699feb89bcfbdadadd\n","Successfully built zmq smart-open gast\n","\u001b[31mERROR: spacy 2.2.4 has requirement tqdm<5.0.0,>=4.38.0, but you'll have tqdm 4.32.2 which is incompatible.\u001b[0m\n","\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n","\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n","Installing collected packages: zmq, bert-tensorflow, numpy, sentencepiece, tensorflow-hub, smart-open, tensorflow-estimator, tensorboard, gast, tensorflow, tqdm, keras, embedding-as-service\n"," Found existing installation: numpy 1.18.2\n"," Uninstalling numpy-1.18.2:\n"," Successfully uninstalled numpy-1.18.2\n"," Found existing installation: tensorflow-hub 0.8.0\n"," Uninstalling tensorflow-hub-0.8.0:\n"," Successfully uninstalled tensorflow-hub-0.8.0\n"," Found existing installation: smart-open 1.11.1\n"," Uninstalling smart-open-1.11.1:\n"," Successfully uninstalled smart-open-1.11.1\n"," Found existing installation: tensorflow-estimator 2.2.0rc0\n"," Uninstalling tensorflow-estimator-2.2.0rc0:\n"," Successfully uninstalled tensorflow-estimator-2.2.0rc0\n"," Found existing installation: tensorboard 2.2.0\n"," Uninstalling tensorboard-2.2.0:\n"," Successfully uninstalled tensorboard-2.2.0\n"," Found existing installation: gast 0.3.3\n"," Uninstalling gast-0.3.3:\n"," Successfully uninstalled gast-0.3.3\n"," Found existing installation: tensorflow 2.2.0rc3\n"," Uninstalling tensorflow-2.2.0rc3:\n"," Successfully uninstalled tensorflow-2.2.0rc3\n"," Found existing installation: tqdm 4.38.0\n"," Uninstalling tqdm-4.38.0:\n"," Successfully uninstalled tqdm-4.38.0\n"," Found existing installation: Keras 2.3.1\n"," Uninstalling Keras-2.3.1:\n"," Successfully uninstalled Keras-2.3.1\n","Successfully installed bert-tensorflow-1.0.1 embedding-as-service-2.0.0 gast-0.2.2 keras-2.2.4 numpy-1.16.4 sentencepiece-0.1.85 smart-open-1.8.4 tensorboard-1.15.0 tensorflow-1.15.0 tensorflow-estimator-1.15.1 tensorflow-hub-0.4.0 tqdm-4.32.2 zmq-0.0.0\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.colab-display-data+json":{"pip_warning":{"packages":["numpy","tqdm"]}}},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"id":"1IJI3jL1r9k6","colab_type":"code","outputId":"6fc8b10c-fcaa-453a-9378-0dea810e0916","executionInfo":{"status":"ok","timestamp":1587356090689,"user_tz":240,"elapsed":126784,"user":{"displayName":"Praxal Suresh Patel","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjZDJ75FrNT9-0GPWBNlo9njpv7eqdcVnrdgzHY=s64","userId":"09765785019516469778"}},"colab":{"base_uri":"https://localhost:8080/","height":122}},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n","\n","Enter your authorization code:\n","··········\n","Mounted at /content/drive\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"AAUwaRyBsG1J","colab_type":"code","outputId":"697c36d2-a838-4192-a244-53c9863cfb48","executionInfo":{"status":"error","timestamp":1587356405707,"user_tz":240,"elapsed":222,"user":{"displayName":"Praxal Suresh Patel","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjZDJ75FrNT9-0GPWBNlo9njpv7eqdcVnrdgzHY=s64","userId":"09765785019516469778"}},"colab":{"base_uri":"https://localhost:8080/","height":409}},"source":["url = '/content/drive/My Drive/all_import.csv'\n","import pandas as pd\n","df2 = pd.read_csv(url)\n"],"execution_count":0,"outputs":[{"output_type":"error","ename":"FileNotFoundError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0murl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'/content/drive/My Drive/all_import.csv'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mdf2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m 674\u001b[0m )\n\u001b[1;32m 675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 676\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 678\u001b[0m \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 446\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 447\u001b[0m \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 448\u001b[0;31m \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp_or_buf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 449\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 450\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m 878\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 879\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 880\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 881\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 882\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m 1112\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"c\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1113\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"c\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1114\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1115\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1116\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"python\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m 1889\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"usecols\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1890\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1891\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1892\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1893\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n","\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n","\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] File /content/drive/My Drive/all_import.csv does not exist: '/content/drive/My Drive/all_import.csv'"]}]},{"cell_type":"code","metadata":{"id":"nLtnI7rh0kHV","colab_type":"code","colab":{}},"source":["df2 = pd.read_csv('/content/drive/My Drive/pfull_text.txt')"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"5ZXVp22M0uN3","colab_type":"code","outputId":"27156683-9105-412c-94f3-4f89946c375a","executionInfo":{"status":"ok","timestamp":1587356563443,"user_tz":240,"elapsed":230,"user":{"displayName":"Praxal Suresh Patel","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjZDJ75FrNT9-0GPWBNlo9njpv7eqdcVnrdgzHY=s64","userId":"09765785019516469778"}},"colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["df2.head()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
indextext
010001244The London trio are up for best UK act and bes...
110138849The biologist and entrepreneur turned the effo...
210145459Australia 's foreign minister said these were ...
310169745For 70 years , my father has lived with the sh...
410201300Comrade Akaash 's statement comes after the re...
\n","
"],"text/plain":[" index text\n","0 10001244 The London trio are up for best UK act and bes...\n","1 10138849 The biologist and entrepreneur turned the effo...\n","2 10145459 Australia 's foreign minister said these were ...\n","3 10169745 For 70 years , my father has lived with the sh...\n","4 10201300 Comrade Akaash 's statement comes after the re..."]},"metadata":{"tags":[]},"execution_count":12}]},{"cell_type":"code","metadata":{"id":"mpmsyFWXtink","colab_type":"code","colab":{}},"source":["df2list = df2.text.to_list()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"_ZSgwrju0xe-","colab_type":"code","colab":{}},"source":["df2list"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"WDyu1syuwhXo","colab_type":"code","outputId":"32eaba31-4e06-438d-e459-0e55abb3779d","executionInfo":{"status":"ok","timestamp":1587344564351,"user_tz":240,"elapsed":228,"user":{"displayName":"Parthvi Shah","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gj14forhnYAb8tD0SHMMufGgXmEIpTA6BXAnPZ5Uvg=s64","userId":"12920190590086167612"}},"colab":{"base_uri":"https://localhost:8080/","height":54}},"source":["df2list[0]"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["\"The London trio are up for best UK act and best album , as well as getting two nominations in the best song category .\\n`` We got told like this morning ` Oh I think you 're nominated ' '' , said Dappy .\\n`` And I was like ` Oh yeah , which one ? '\\nAnd now we 've got nominated for four awards .\\nI mean , wow ! ''\\nBandmate Fazer added : `` We thought it 's best of us to come down and mingle with everyone and say hello to the cameras .\\nAnd now we find we 've got four nominations . ''\\nThe band have two shots at the best song prize , getting the nod for their Tynchy Stryder collaboration Number One , and single Strong Again .\\nTheir album Uncle B will also go up against records by the likes of Beyonce and Kanye West .\\nN-Dubz picked up the best newcomer Mobo in 2007 , but female member Tulisa said they would n't be too disappointed if they did n't win this time around .\\n`` At the end of the day we 're grateful to be where we are in our careers .\\n`` If it do n't happen then it do n't happen - live to fight another day and keep on making albums and hits for the fans . ''\\nDappy also revealed they could be performing live several times on the night .\\nThe group will be doing Number One and also a possible rendition of the War Child single , I Got Soul .\\nThe charity song is a re-working of The Killers ' All These Things That I 've Done and is set to feature artists like Chipmunk , Ironik and Pixie Lott .\\nThis year 's Mobos will be held outside of London for the first time , in Glasgow on 30 September .\\nN-Dubz said they were looking forward to performing for their Scottish fans and boasted about their recent shows north of the border .\\n`` We just done Edinburgh the other day , '' said Dappy .\\n`` We smashed up an N-Dubz show over there .\\nWe done Aberdeen about three or four months ago - we smashed up that show over there !\\nEverywhere we go we smash it up ! ''\\n\""]},"metadata":{"tags":[]},"execution_count":19}]},{"cell_type":"code","metadata":{"id":"mCgv9zKos-VM","colab_type":"code","colab":{}},"source":["dftry = df2.iloc[:250,-1]"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"g-yJRBSXtQUt","colab_type":"code","colab":{}},"source":["dftry = dftry.to_list()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"yXr-TzUN1u8R","colab_type":"code","colab":{}},"source":["from embedding_as_service.text.encode import Encoder"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"fvamnyHW1xSb","colab_type":"code","outputId":"75a77ec8-66a7-4d93-dec8-2b441fd97088","executionInfo":{"status":"ok","timestamp":1587356715223,"user_tz":240,"elapsed":8219,"user":{"displayName":"Praxal Suresh Patel","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjZDJ75FrNT9-0GPWBNlo9njpv7eqdcVnrdgzHY=s64","userId":"09765785019516469778"}},"colab":{"base_uri":"https://localhost:8080/","height":236}},"source":["en = Encoder(embedding='xlnet', model='xlnet_base_cased')"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"text/html":["

\n","The default version of TensorFlow in Colab will switch to TensorFlow 2.x on the 27th of March, 2020.
\n","We recommend you upgrade now\n","or ensure your notebook will continue to use TensorFlow 1.x via the %tensorflow_version 1.x magic:\n","more info.

\n"],"text/plain":[""]},"metadata":{"tags":[]}},{"output_type":"stream","text":["WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/embedding_as_service/text/xlnet/__init__.py:62: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n","\n","Model does not exists, Downloading model: xlnet_base_cased\n","[||||||||||||||||||||||||||||||||||||||||||||||||||] 100 % [423474/423474 kb]\n","Model downloaded successfully!\n","Loading Model (this might take few minutes).....\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/embedding_as_service/text/xlnet/models/xlnet.py:62: The name tf.gfile.Open is deprecated. Please use tf.io.gfile.GFile instead.\n","\n","Model loaded Successfully !\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"ClctPcupxOPi","colab_type":"code","colab":{}},"source":["import numpy as np\n","import tensorflow as tf\n","config = tf.ConfigProto()\n","config.gpu_options.allow_growth = True\n"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"uILkhnaP8hiB","colab_type":"code","outputId":"d8eb3034-0864-4ee6-a62c-7f2e71f84c9c","executionInfo":{"status":"ok","timestamp":1587358124008,"user_tz":240,"elapsed":1398620,"user":{"displayName":"Praxal Suresh Patel","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjZDJ75FrNT9-0GPWBNlo9njpv7eqdcVnrdgzHY=s64","userId":"09765785019516469778"}},"colab":{"base_uri":"https://localhost:8080/","height":1000}},"source":["pd1 = (en.encode(df2list))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 416.74it/s]"],"name":"stderr"},{"output_type":"stream","text":["WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/embedding_as_service/text/xlnet/models/xlnet.py:219: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n","\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/embedding_as_service/text/xlnet/models/xlnet.py:219: The name tf.AUTO_REUSE is deprecated. Please use tf.compat.v1.AUTO_REUSE instead.\n","\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/embedding_as_service/text/xlnet/models/modeling.py:453: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n","\n","INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/embedding_as_service/text/xlnet/models/modeling.py:460: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n","\n"],"name":"stdout"},{"output_type":"stream","text":["\n"],"name":"stderr"},{"output_type":"stream","text":["WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/embedding_as_service/text/xlnet/models/modeling.py:535: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use keras.layers.dropout instead.\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/core.py:271: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Please use `layer.__call__` method instead.\n","WARNING:tensorflow:\n","The TensorFlow contrib module will not be included in TensorFlow 2.0.\n","For more information, please see:\n"," * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n"," * https://github.com/tensorflow/addons\n"," * https://github.com/tensorflow/io (for I/O related ops)\n","If you depend on functionality not listed there, please file an issue.\n","\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/embedding_as_service/text/xlnet/models/modeling.py:67: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use keras.layers.Dense instead.\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/embedding_as_service/text/xlnet/__init__.py:148: The name tf.initializers.global_variables is deprecated. Please use tf.compat.v1.initializers.global_variables instead.\n","\n"],"name":"stdout"},{"output_type":"stream","text":["Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 438.14it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 456.86it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 454.82it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 465.65it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 594.40it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 517.24it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 662.41it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 646.80it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 699.25it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 630.30it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 667.44it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 612.01it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 644.94it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n"],"name":"stdout"},{"output_type":"stream","text":["\n"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 567.59it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 602.29it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 549.28it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 607.27it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 513.74it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n"],"name":"stdout"},{"output_type":"stream","text":["\n"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 478.54it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 520.58it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 527.33it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 474.76it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 480.36it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 463.66it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 483.94it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 514.67it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n"],"name":"stdout"},{"output_type":"stream","text":["\n"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 580.75it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 467.05it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 476.85it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 431.58it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 396.29it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 372.93it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 505.28it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 568.15it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 557.57it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 520.89it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 485.03it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 501.48it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 536.95it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 521.68it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 523.63it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 544.09it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 488.80it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 70/70 [00:00<00:00, 466.16it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"02YdJrSv9PPw","colab_type":"code","outputId":"7c3cbe18-63fb-4c62-f91f-2b34c8230442","executionInfo":{"status":"error","timestamp":1587360163617,"user_tz":240,"elapsed":193,"user":{"displayName":"Praxal Suresh Patel","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjZDJ75FrNT9-0GPWBNlo9njpv7eqdcVnrdgzHY=s64","userId":"09765785019516469778"}},"colab":{"base_uri":"https://localhost:8080/","height":351}},"source":["from numpy import asarray\n","from numpy import savetxt\n","# define data\n","\n","# save to csv file\n","savetxt('/content/drive/My Drive/full_text_embeddings.csv', pd1, delimiter=',')"],"execution_count":0,"outputs":[{"output_type":"error","ename":"ValueError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;31m# save to csv file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0msavetxt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/My Drive/full_text_embeddings.csv'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpd1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdelimiter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m','\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m<__array_function__ internals>\u001b[0m in \u001b[0;36msavetxt\u001b[0;34m(*args, **kwargs)\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/numpy/lib/npyio.py\u001b[0m in \u001b[0;36msavetxt\u001b[0;34m(fname, X, fmt, delimiter, newline, header, footer, comments, encoding)\u001b[0m\n\u001b[1;32m 1393\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfmt\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1394\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfmt\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mncol\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1395\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'fmt has wrong shape. %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfmt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1396\u001b[0m \u001b[0mformat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0masstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdelimiter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0masstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfmt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1397\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfmt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mValueError\u001b[0m: Expected 1D or 2D array, got 3D array instead"]}]},{"cell_type":"code","metadata":{"id":"gvSBRxoQCq9H","colab_type":"code","colab":{}},"source":["\n","# save numpy array as npy file\n","from numpy import asarray\n","from numpy import save\n","# define data\n","\n","# save to npy file\n","save('/content/drive/My Drive/full_text_embeddings.csv.npy', pd1)\n"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Tjx_NlWDwAbt","colab_type":"text"},"source":["**Get outputs from intermediate layers from BERT**"]},{"cell_type":"code","metadata":{"id":"yefw8LsK179g","colab_type":"code","colab":{}},"source":["pd2 = pd.read_csv('/content/drive/My Drive/pmodel_preds.txt')"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"NQEqcqa5v7hg","colab_type":"code","colab":{}},"source":["\"\"\"\n","1. Clone the repo\n","2. Open embedding-as-service/embedding_as_service/text/bert/__init__.py\n","3. Replace the following code - \n","```\n"," bert_outputs = self.bert_module(bert_inputs, signature=\"tokens\", as_dict=True)\n"," sequence_output = bert_outputs[\"sequence_output\"]\n"," token_embeddings = self.sess.run(sequence_output)\n","```\n"," with - \n","```\n"," bert_outputs = self.bert_module(bert_inputs, signature=\"tokens\", as_dict=True)\n"," sequence_output = bert_outputs[\"sequence_output\"]\n"," sequence_output = get_intermediate_layer(last_layer=sequence_output,\n"," total_layers=12,\n"," desired_layer=6)\n"," token_embeddings = self.sess.run(sequence_output)\n","```\n","NOTE - I used total_layers = 12 and desired_layer = 6 because in using \n"," BERT-base-uncased which has 12 layers and i needed output from the \n"," 7th layer.\n"," `get_intermediate_layer` is defined below.\n","\"\"\""],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"1058I8BbGBp4","colab_type":"code","colab":{}},"source":["gold_std = pd2['gold'].to_list()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"oSayEiytGVJ6","colab_type":"code","outputId":"7e295e1b-b2df-4d90-af22-be011e792c17","executionInfo":{"status":"ok","timestamp":1587363024264,"user_tz":240,"elapsed":326163,"user":{"displayName":"Praxal Suresh Patel","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjZDJ75FrNT9-0GPWBNlo9njpv7eqdcVnrdgzHY=s64","userId":"09765785019516469778"}},"colab":{"base_uri":"https://localhost:8080/","height":1000}},"source":["pd_gold = (en.encode(gold_std))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 6795.06it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7320.25it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7934.54it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7355.40it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7038.48it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 8124.93it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 6882.52it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7867.97it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 6790.03it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7017.00it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7364.28it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7617.89it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7379.21it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7181.84it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7153.51it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7642.24it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 8077.13it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 8326.16it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 8011.03it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7789.26it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7724.98it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7661.43it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7181.45it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 6905.36it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 8106.65it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7885.59it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7779.78it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 6514.95it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7628.12it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7278.47it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7800.86it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7483.88it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7384.08it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7263.21it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 8145.95it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7852.03it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7751.86it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7857.32it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7393.49it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7929.27it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7381.09it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 6607.60it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 7710.78it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 256/256 [00:00<00:00, 6801.95it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n","Converting texts to features: 100%|██████████| 70/70 [00:00<00:00, 6162.66it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:memory input None\n","INFO:tensorflow:Use float type \n"],"name":"stdout"},{"output_type":"stream","text":["\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"3re6qm2uGpEt","colab_type":"code","colab":{}},"source":["save('/content/drive/My Drive/gold_std_embeddings.csv.npy', pd_gold)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"LHsmqEZyGhGu","colab_type":"code","colab":{}},"source":["convs2s_std = pd2['convs2s'].to_list()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"nI516uo7Gz8J","colab_type":"code","colab":{}},"source":["pd_convs2s = (en.encode(convs2s_std))"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"-01FDiFAG7SK","colab_type":"code","colab":{}},"source":["save('/content/drive/My Drive/conv2s_std_embeddings.csv.npy', pd_convs2s )"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"GGzUrHmmHEVH","colab_type":"code","colab":{}},"source":["convs2s_std = pd2['oracle'].to_list()\n","pd_oracle = (en.encode(oracle_std))\n","save('/content/drive/My Drive/oracle_embeddings.csv.npy', pd_oracle)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"7oijD_9uxiJG","colab_type":"code","colab":{}},"source":["def get_intermediate_layer(last_layer, total_layers, desired_layer):\n"," \"\"\"\n"," Method to get outputs from any intermediate layer of bert model.\n"," \"\"\"\n"," intermediate_layer_name = last_layer.name.replace(str(total_layers + 1),\n"," str(desired_layer + 1))\n"," return tf.get_default_graph().get_tensor_by_name(intermediate_layer_name)"],"execution_count":0,"outputs":[]}]} \ No newline at end of file