diff --git a/notebooks/Simple_Transformer_Language_Model.ipynb b/notebooks/Simple_Transformer_Language_Model.ipynb index d887ce65849f3..5ee13be3abf1d 100644 --- a/notebooks/Simple_Transformer_Language_Model.ipynb +++ b/notebooks/Simple_Transformer_Language_Model.ipynb @@ -1,343 +1 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "Simple Transformer Language Model.ipynb", - "provenance": [], - "collapsed_sections": [], - "authorship_tag": "ABX9TyOEWirarSiFWRClhLdkaXby", - "include_colab_link": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "view-in-github", - "colab_type": "text" - }, - "source": [ - "\"Open" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "SfaGYwr6QGl0", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 360 - }, - "outputId": "45caa369-4bda-420d-f407-1002204c6e83" - }, - "source": [ - "!pip install transformers" - ], - "execution_count": 103, - "outputs": [ - { - "output_type": "stream", - "text": [ - "Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (3.1.0)\n", - "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)\n", - "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n", - "Requirement already satisfied: tokenizers==0.8.1.rc2 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8.1rc2)\n", - "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n", - "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n", - "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n", - "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n", - "Requirement already satisfied: sentencepiece!=0.1.92 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.1.91)\n", - "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n", - "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n", - "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n", - "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n", - "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n", - "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n", - "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n", - "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n" - ], - "name": "stdout" - } - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "SKOHXyzNQ5Wa", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from transformers import AutoTokenizer, AutoModelForCausalLM\n", - "tokenizer = AutoTokenizer.from_pretrained(\"distilgpt2\") \n", - "model = AutoModelForCausalLM.from_pretrained(\"distilgpt2\", output_hidden_states=True)" - ], - "execution_count": 104, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "zdQBaZZcQ_Sa", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 68 - }, - "outputId": "4cc3dbf5-922d-4eda-b47e-75e8adbc96e2" - }, - "source": [ - "text = \"The Shawshank\"\n", - "\n", - "# Tokenize the input string\n", - "input = tokenizer.encode(text, return_tensors=\"pt\")\n", - "\n", - "# Run the model\n", - "output = model.generate(input, max_length=5, do_sample=False)\n", - "\n", - "# Print the output\n", - "print('\\n',tokenizer.decode(output[0]))" - ], - "execution_count": 105, - "outputs": [ - { - "output_type": "stream", - "text": [ - "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n" - ], - "name": "stderr" - }, - { - "output_type": "stream", - "text": [ - "\n", - " The Shawshank Redemption\n" - ], - "name": "stdout" - } - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "m8m1rK3gNNW4", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 - }, - "outputId": "61da2785-cd28-4b00-84bf-ac65deea8d57" - }, - "source": [ - "# Print the token ides (of the input and output)\n", - "output" - ], - "execution_count": 101, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "tensor([[ 464, 18193, 1477, 962, 34433]])" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 101 - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "tProbSeTJATA", - "colab_type": "text" - }, - "source": [ - "## From words to vectors and back" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Dy2Pjd--ROa5", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 - }, - "outputId": "1227d405-a6b1-48bd-f6e6-127ea739cc04" - }, - "source": [ - "# Print the input token ids\n", - "text = \"The Shawshank\"\n", - "input = tokenizer(text, return_tensors=\"pt\")['input_ids']\n", - "input" - ], - "execution_count": 102, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "tensor([[ 464, 18193, 1477, 962]])" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 102 - } - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "sTGffdCOJbdo", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 - }, - "outputId": "e45abbde-d575-40bb-82b8-48dc2c90ffb3" - }, - "source": [ - "tokenizer.convert_ids_to_tokens(input[0])" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "['The', 'ĠShaw', 'sh', 'ank']" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 91 - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4lbn94y0P2UV", - "colab_type": "text" - }, - "source": [ - "## Breathe meaning into numbers (Embedding)" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "5QCFBcxZQIN8", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 - }, - "outputId": "78f4012c-3c7c-4c89-813a-62abae5cd98d" - }, - "source": [ - "# This is the embedding matrix of the model\n", - "model.transformer.wte # Dimensions are: (Number of tokens in vocabulary, dimension of model)" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "Embedding(50257, 768)" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 95 - } - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "0Ah9tc1gP7lX", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# Get the embedding vector of token # 464 ('The')\n", - "model.transformer.wte(torch.tensor(464))" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "ZT5lmGVK60mJ", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 68 - }, - "outputId": "3c82374f-eee5-4e19-ef64-e0908b5719ee" - }, - "source": [ - "text = \"The chicken didn't cross the road because it was\"\n", - "\n", - "# Tokenize the input string\n", - "input = tokenizer.encode(text, return_tensors=\"pt\")\n", - "\n", - "# Run the model\n", - "output = model.generate(input, max_length=20, do_sample=True)\n", - "\n", - "# Print the output\n", - "print('\\n',tokenizer.decode(output[0]))" - ], - "execution_count": 108, - "outputs": [ - { - "output_type": "stream", - "text": [ - "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n" - ], - "name": "stderr" - }, - { - "output_type": "stream", - "text": [ - "\n", - " The chicken didn't cross the road because it was in the street. The driver then asked if the\n" - ], - "name": "stdout" - } - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "BstYQU6NkkDA", - "colab_type": "code", - "colab": {} - }, - "source": [ - "" - ], - "execution_count": null, - "outputs": [] - } - ] -} \ No newline at end of file +{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Simple_Transformer_Language_Model.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"d724d7b5f9594451b39920745d291a27":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_7776aeb5b6a544a0ac4be74a1554880f","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_33c3721b713c4ac9a06f3b7fa38c71f1","IPY_MODEL_93534b5f8ca54a95a04f92f95a384217"]}},"7776aeb5b6a544a0ac4be74a1554880f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"33c3721b713c4ac9a06f3b7fa38c71f1":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_6003ada563764d4a8474ac7f6ba81e78","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":762,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":762,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_e31d7ec37f2d4647a4f08c43a74b8799"}},"93534b5f8ca54a95a04f92f95a384217":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_a2fd9d706e554fefae73bef9fe2ff6c9","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 762/762 [00:00<00:00, 12.0kB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_1c611260a12a49c58d68913475f6966c"}},"6003ada563764d4a8474ac7f6ba81e78":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"e31d7ec37f2d4647a4f08c43a74b8799":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"a2fd9d706e554fefae73bef9fe2ff6c9":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"1c611260a12a49c58d68913475f6966c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ef96b811798f4408a9922c9521550fbc":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_71c1bbb377a54a3d95a2b7b1281bb0e6","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_4dcb9331c54d4734afe93cb51dbff74b","IPY_MODEL_9cb7be0d325a4d7a92af7615635feca8"]}},"71c1bbb377a54a3d95a2b7b1281bb0e6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"4dcb9331c54d4734afe93cb51dbff74b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_23efab17b3d74e61815998a6a11d7354","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":1042301,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1042301,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_6ec16c4ffdd142f69f67a2b7b9e9ca50"}},"9cb7be0d325a4d7a92af7615635feca8":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_4b534862b4d84283bac226b844658cf2","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1.04M/1.04M [00:00<00:00, 2.65MB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_76a4178d6f6b4f77acdf1cd8cb5ff464"}},"23efab17b3d74e61815998a6a11d7354":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"6ec16c4ffdd142f69f67a2b7b9e9ca50":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"4b534862b4d84283bac226b844658cf2":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"76a4178d6f6b4f77acdf1cd8cb5ff464":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"4ea7732b969e4f10ab2f46d4755cd8d9":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_f6c8ada41be146b381209b97646dbc75","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_3039714cf753464e97bfd23b0f2f7f30","IPY_MODEL_c831151e3bf94525a64a02cdb0b295e7"]}},"f6c8ada41be146b381209b97646dbc75":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3039714cf753464e97bfd23b0f2f7f30":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_48716d9b96f44603af36417ccfc94412","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":456318,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":456318,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_4e1d46605a764ab9a5a0c76df9142d8d"}},"c831151e3bf94525a64a02cdb0b295e7":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_3bafe17b92bf4f9d80a12043a156cf06","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 456k/456k [00:00<00:00, 1.57MB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_2c4a2a5bb56d423fb06c2b8bdf77851b"}},"48716d9b96f44603af36417ccfc94412":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"4e1d46605a764ab9a5a0c76df9142d8d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3bafe17b92bf4f9d80a12043a156cf06":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"2c4a2a5bb56d423fb06c2b8bdf77851b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3c39f01231594caca8355e20ffba6ab8":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_9d9d93eaa31b42088a185c5217899fae","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_c16fb0fe42ee4d73ac3f9c9dc53b41be","IPY_MODEL_a1707899962f41beb3141fc352170c48"]}},"9d9d93eaa31b42088a185c5217899fae":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"c16fb0fe42ee4d73ac3f9c9dc53b41be":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_b59e4f3a8298418894abbe72c9d10017","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":1355256,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1355256,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_eb67d7be345644dc801b625edbe236de"}},"a1707899962f41beb3141fc352170c48":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_ce9a7558fc164cd2a5625940ca819a3e","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1.36M/1.36M [00:00<00:00, 7.47MB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_478a9dbd380e426283fbf800bf6af162"}},"b59e4f3a8298418894abbe72c9d10017":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"eb67d7be345644dc801b625edbe236de":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ce9a7558fc164cd2a5625940ca819a3e":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"478a9dbd380e426283fbf800bf6af162":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"27df51031a2c412e9571b238118dd9f2":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_804becb2d1c54c7090a8eb0bb741fc39","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_8d896333bd9d4cbdbc1adeb7335d7a2e","IPY_MODEL_e3b71de1bc894daabaf47e29d2b75b12"]}},"804becb2d1c54c7090a8eb0bb741fc39":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"8d896333bd9d4cbdbc1adeb7335d7a2e":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_37f6a63a7b514a249462635ae75ab488","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":352833716,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":352833716,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_e02c120fc74043bab161d6e1c5cd509b"}},"e3b71de1bc894daabaf47e29d2b75b12":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_14fba9d627104943bf267fb46d97b419","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 353M/353M [00:05<00:00, 61.7MB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_62597e305b9e4e46a19c7975d26eba75"}},"37f6a63a7b514a249462635ae75ab488":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"e02c120fc74043bab161d6e1c5cd509b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"14fba9d627104943bf267fb46d97b419":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"62597e305b9e4e46a19c7975d26eba75":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"cells":[{"cell_type":"markdown","metadata":{"id":"view-in-github"},"source":["\"Open"]},{"cell_type":"code","metadata":{"id":"SfaGYwr6QGl0","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1613206366501,"user_tz":-540,"elapsed":6512,"user":{"displayName":"­권태확","photoUrl":"","userId":"00214940863527742140"}},"outputId":"02ec679b-6623-45c6-bc4e-1b70819d779c"},"source":["!pip install transformers"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Collecting transformers\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/98/87/ef312eef26f5cecd8b17ae9654cdd8d1fae1eb6dbd87257d6d73c128a4d0/transformers-4.3.2-py3-none-any.whl (1.8MB)\n","\u001b[K |████████████████████████████████| 1.8MB 15.9MB/s \n","\u001b[?25hRequirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n","Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from transformers) (3.4.0)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n","Collecting tokenizers<0.11,>=0.10.1\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/fd/5b/44baae602e0a30bcc53fbdbc60bd940c15e143d252d658dfdefce736ece5/tokenizers-0.10.1-cp36-cp36m-manylinux2010_x86_64.whl (3.2MB)\n","\u001b[K |████████████████████████████████| 3.2MB 55.4MB/s \n","\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.19.5)\n","Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n","Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.9)\n","Collecting sacremoses\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n","\u001b[K |████████████████████████████████| 890kB 57.7MB/s \n","\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.0)\n","Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.12.5)\n","Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n","Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n","Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.0.0)\n","Building wheels for collected packages: sacremoses\n"," Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n"," Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=e81a13023c308fd44874c762b4f51154b950e0d27d24cb6d823a80dd16036aba\n"," Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n","Successfully built sacremoses\n","Installing collected packages: tokenizers, sacremoses, transformers\n","Successfully installed sacremoses-0.0.43 tokenizers-0.10.1 transformers-4.3.2\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"SKOHXyzNQ5Wa","colab":{"base_uri":"https://localhost:8080/","height":267,"referenced_widgets":["d724d7b5f9594451b39920745d291a27","7776aeb5b6a544a0ac4be74a1554880f","33c3721b713c4ac9a06f3b7fa38c71f1","93534b5f8ca54a95a04f92f95a384217","6003ada563764d4a8474ac7f6ba81e78","e31d7ec37f2d4647a4f08c43a74b8799","a2fd9d706e554fefae73bef9fe2ff6c9","1c611260a12a49c58d68913475f6966c","ef96b811798f4408a9922c9521550fbc","71c1bbb377a54a3d95a2b7b1281bb0e6","4dcb9331c54d4734afe93cb51dbff74b","9cb7be0d325a4d7a92af7615635feca8","23efab17b3d74e61815998a6a11d7354","6ec16c4ffdd142f69f67a2b7b9e9ca50","4b534862b4d84283bac226b844658cf2","76a4178d6f6b4f77acdf1cd8cb5ff464","4ea7732b969e4f10ab2f46d4755cd8d9","f6c8ada41be146b381209b97646dbc75","3039714cf753464e97bfd23b0f2f7f30","c831151e3bf94525a64a02cdb0b295e7","48716d9b96f44603af36417ccfc94412","4e1d46605a764ab9a5a0c76df9142d8d","3bafe17b92bf4f9d80a12043a156cf06","2c4a2a5bb56d423fb06c2b8bdf77851b","3c39f01231594caca8355e20ffba6ab8","9d9d93eaa31b42088a185c5217899fae","c16fb0fe42ee4d73ac3f9c9dc53b41be","a1707899962f41beb3141fc352170c48","b59e4f3a8298418894abbe72c9d10017","eb67d7be345644dc801b625edbe236de","ce9a7558fc164cd2a5625940ca819a3e","478a9dbd380e426283fbf800bf6af162","27df51031a2c412e9571b238118dd9f2","804becb2d1c54c7090a8eb0bb741fc39","8d896333bd9d4cbdbc1adeb7335d7a2e","e3b71de1bc894daabaf47e29d2b75b12","37f6a63a7b514a249462635ae75ab488","e02c120fc74043bab161d6e1c5cd509b","14fba9d627104943bf267fb46d97b419","62597e305b9e4e46a19c7975d26eba75"]},"executionInfo":{"status":"ok","timestamp":1613206380990,"user_tz":-540,"elapsed":20993,"user":{"displayName":"­권태확","photoUrl":"","userId":"00214940863527742140"}},"outputId":"7c74f600-20c4-4aa4-a854-892e96216831"},"source":["from transformers import AutoTokenizer, AutoModelForCausalLM\n","tokenizer = AutoTokenizer.from_pretrained(\"distilgpt2\") \n","model = AutoModelForCausalLM.from_pretrained(\"distilgpt2\", output_hidden_states=True)"],"execution_count":2,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"d724d7b5f9594451b39920745d291a27","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=762.0, style=ProgressStyle(description_…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"ef96b811798f4408a9922c9521550fbc","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1042301.0, style=ProgressStyle(descript…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"4ea7732b969e4f10ab2f46d4755cd8d9","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"3c39f01231594caca8355e20ffba6ab8","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1355256.0, style=ProgressStyle(descript…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"27df51031a2c412e9571b238118dd9f2","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=352833716.0, style=ProgressStyle(descri…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"zdQBaZZcQ_Sa","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1613206380999,"user_tz":-540,"elapsed":20991,"user":{"displayName":"­권태확","photoUrl":"","userId":"00214940863527742140"}},"outputId":"da4663b0-f911-4a09-e510-0578965ba825"},"source":["text = \"The Shawshank\"\n","\n","# Tokenize the input string\n","input = tokenizer.encode(text, return_tensors=\"pt\")\n","\n","# Run the model\n","output = model.generate(input, max_length=5, do_sample=False)\n","\n","# Print the output\n","print('\\n',tokenizer.decode(output[0]))"],"execution_count":3,"outputs":[{"output_type":"stream","text":["Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"],"name":"stderr"},{"output_type":"stream","text":["\n"," The Shawshank Redemption\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"m8m1rK3gNNW4","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1613206381001,"user_tz":-540,"elapsed":20985,"user":{"displayName":"­권태확","photoUrl":"","userId":"00214940863527742140"}},"outputId":"2c06cb13-8b3f-465d-e0bc-0b5e4e6e0557"},"source":["# Print the token ides (of the input and output)\n","output"],"execution_count":4,"outputs":[{"output_type":"execute_result","data":{"text/plain":["tensor([[ 464, 18193, 1477, 962, 34433]])"]},"metadata":{"tags":[]},"execution_count":4}]},{"cell_type":"markdown","metadata":{"id":"tProbSeTJATA"},"source":["## From words to vectors and back"]},{"cell_type":"code","metadata":{"id":"Dy2Pjd--ROa5","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1613206381002,"user_tz":-540,"elapsed":20977,"user":{"displayName":"­권태확","photoUrl":"","userId":"00214940863527742140"}},"outputId":"eec16df0-7a98-41e2-837c-a88f3e5f0ec4"},"source":["# Print the input token ids\n","text = \"The Shawshank\"\n","input = tokenizer(text, return_tensors=\"pt\")['input_ids']\n","input"],"execution_count":5,"outputs":[{"output_type":"execute_result","data":{"text/plain":["tensor([[ 464, 18193, 1477, 962]])"]},"metadata":{"tags":[]},"execution_count":5}]},{"cell_type":"code","metadata":{"id":"sTGffdCOJbdo","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1613206381004,"user_tz":-540,"elapsed":20970,"user":{"displayName":"­권태확","photoUrl":"","userId":"00214940863527742140"}},"outputId":"fe046f8d-079f-4c3b-d549-185e82c3a4c8"},"source":["tokenizer.convert_ids_to_tokens(input[0])"],"execution_count":6,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['The', 'ĠShaw', 'sh', 'ank']"]},"metadata":{"tags":[]},"execution_count":6}]},{"cell_type":"markdown","metadata":{"id":"4lbn94y0P2UV"},"source":["## Breathe meaning into numbers (Embedding)"]},{"cell_type":"code","metadata":{"id":"5QCFBcxZQIN8","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1613206381005,"user_tz":-540,"elapsed":20961,"user":{"displayName":"­권태확","photoUrl":"","userId":"00214940863527742140"}},"outputId":"c19c3bc8-e5ad-4bfd-94c9-90d37258b709"},"source":["# This is the embedding matrix of the model\n","model.transformer.wte # Dimensions are: (Number of tokens in vocabulary, dimension of model)"],"execution_count":7,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Embedding(50257, 768)"]},"metadata":{"tags":[]},"execution_count":7}]},{"cell_type":"code","metadata":{"id":"0Ah9tc1gP7lX","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1613206381007,"user_tz":-540,"elapsed":20960,"user":{"displayName":"­권태확","photoUrl":"","userId":"00214940863527742140"}},"outputId":"dd010290-32ba-4eeb-fb46-68906711cd1a"},"source":["# Get the embedding vector of token # 464 ('The')\n","import torch\n","model.transformer.wte(torch.tensor(464))"],"execution_count":8,"outputs":[{"output_type":"execute_result","data":{"text/plain":["tensor([-6.2649e-02, -4.4906e-02, 5.5888e-02, -5.4657e-02, -1.1713e-01,\n"," -7.2870e-02, -2.2326e-01, -3.2198e-03, 6.8535e-03, 2.3608e-02,\n"," -5.8700e-02, 4.4439e-02, 7.4774e-02, -1.3818e-02, 1.1873e-01,\n"," -5.1842e-02, 5.4415e-02, 5.5382e-02, -3.3126e-02, 1.1923e-01,\n"," -7.3283e-02, 2.6658e-02, -8.4261e-02, 5.7980e-02, -4.2860e-03,\n"," -4.0704e-02, 6.5652e-02, -6.6221e-02, -1.0232e-01, 3.1356e-02,\n"," -1.8873e-02, 2.7774e-02, -2.0423e-02, 1.1994e-01, -8.7572e-02,\n"," -1.0579e-01, -3.1816e-01, 9.5807e-02, 1.1588e-01, -4.2873e-02,\n"," 1.3187e-01, -1.3457e-01, -1.0421e-01, -1.2150e-01, 9.2551e-02,\n"," -2.7394e-02, 3.1406e-02, 6.4891e-03, 1.2296e-01, -2.0581e-01,\n"," -6.3499e-02, 5.4726e-02, 6.0272e-02, 1.1968e-01, 6.4859e-02,\n"," -3.4885e-01, -5.6065e-02, -2.4721e-03, 3.7549e-03, -1.2817e-02,\n"," -8.4894e-02, -1.6269e-02, 8.2583e-02, -4.0134e-02, -2.1432e-01,\n"," -2.6889e-03, 1.5291e-02, 5.2090e-02, 2.9089e-01, 7.6877e-02,\n"," 6.9082e-02, 7.3376e-02, 8.8179e-03, 9.6406e-02, 2.0432e-02,\n"," -7.4213e-02, -4.1877e-02, 7.1981e-02, -4.8161e-02, 1.1771e-02,\n"," 1.0166e-01, -5.4755e-02, 8.7261e-02, -7.9654e-02, 5.2315e-01,\n"," -9.9835e-02, -1.0481e-01, -2.8195e-01, 8.1683e-03, -4.4626e-02,\n"," -6.5149e-02, 7.7208e-02, -9.7715e-02, -5.6882e-02, 1.0190e-01,\n"," 1.2256e-02, -2.2082e-02, -4.6576e-02, 8.7919e-02, 2.1517e-02,\n"," 1.6148e-01, 6.5791e-02, -8.3689e-02, -7.6088e-02, -8.4335e-02,\n"," -9.1179e-02, -4.1727e-02, -1.1803e-01, 1.4415e-01, -7.8215e-02,\n"," -5.9557e-03, -3.4881e-02, -5.6165e-02, 2.1938e-02, -8.0716e-02,\n"," 1.2922e-01, 1.5276e-01, 1.2612e-01, 1.0735e-01, 3.9891e-02,\n"," -7.1642e-03, 7.1192e-02, -1.2086e-01, -6.3376e-02, -1.7415e-01,\n"," 1.1116e-01, -1.4673e-01, -6.5523e-03, 7.3890e-02, -4.9352e-02,\n"," 3.0207e-02, -1.0208e-01, 6.2928e-02, 1.8352e-01, -4.9800e-02,\n"," -2.7031e-02, 1.4378e-03, -5.3656e-02, 6.2120e-01, -9.3976e-02,\n"," 5.1804e-02, -2.2970e-02, 6.7755e-02, 1.8769e-02, 2.6562e-02,\n"," -1.0017e-01, -1.7162e-02, -2.6100e-03, 1.4294e-02, 4.4294e-02,\n"," 1.3903e-01, -1.4014e-02, 2.3181e-02, -1.4834e-01, -5.3028e-02,\n"," 9.6540e-02, 2.3293e-02, -7.7461e-02, -1.0504e-01, -3.5051e-01,\n"," -3.7689e-02, 1.0662e-02, -1.2818e-01, -6.2468e-02, 4.2055e-02,\n"," 2.1855e-03, -1.4826e-01, 2.3176e-01, 3.3421e-02, 2.2398e-01,\n"," 1.1421e-01, 3.2895e-02, -3.0494e-02, -5.3284e-02, 1.0991e-01,\n"," 3.5463e-02, 2.7445e-01, 6.2911e-04, 7.8582e-02, 1.0000e-01,\n"," 2.4097e-02, -9.4483e-02, 9.1651e-02, 3.0298e-03, 1.4840e-01,\n"," -1.7038e-02, -1.1124e-01, 4.0140e-03, -4.1108e-04, -1.1162e-01,\n"," 5.6105e-02, 1.8361e-01, 3.6446e-02, 9.7009e-02, -2.1477e-01,\n"," 9.2157e-02, 3.5488e-02, 1.6611e-02, -6.8011e-02, 1.6890e-01,\n"," 2.4187e-02, -9.6880e-02, -9.7266e-02, -6.0271e-02, 7.9955e-02,\n"," 9.8765e-04, 5.0787e-03, 7.2527e-02, 1.1458e-01, 1.8893e-02,\n"," 9.9052e-02, -6.2180e-02, -1.9884e-03, -5.9710e-02, 1.0698e-01,\n"," -4.7192e-02, -9.0067e-02, 5.1061e-02, 6.7179e-03, -1.0657e-01,\n"," 1.1859e-01, -1.6188e-02, -8.5031e-02, -2.8055e-02, -4.9350e-02,\n"," -1.0238e-01, -6.7360e-02, -1.0790e-01, 8.8089e-02, -1.2654e-01,\n"," 1.4105e-01, 1.3072e-01, -2.5147e-01, 3.3939e-02, -9.8332e-02,\n"," -5.6578e-02, 1.2558e-02, -6.2258e-02, -2.6592e-02, -1.3017e-02,\n"," 1.0194e-01, -1.1762e-03, 7.7183e-02, 5.7278e-02, 7.7125e-02,\n"," -3.6276e-02, 4.7724e-02, -2.0229e-02, -1.7146e-02, 9.1073e-02,\n"," 1.7726e-01, -9.0652e-02, -9.3985e-02, -6.0687e-02, 6.2885e-02,\n"," -1.6106e-02, 1.2610e-02, 3.5395e-02, 9.8192e-02, 7.1861e-02,\n"," 7.1699e-02, 9.4895e-02, 3.8220e-02, 1.3879e-01, 3.7747e-02,\n"," -8.2909e-02, -2.8665e-01, 2.2522e-02, 7.0700e-02, -1.2517e-01,\n"," 1.8129e-01, -1.1479e-01, -3.0415e-02, -4.4267e-02, 1.1545e-01,\n"," -1.0746e-01, -4.6373e-02, 9.9404e-02, -8.4753e-02, 5.8346e-02,\n"," 7.6724e-04, -6.9820e-02, -1.2692e-01, -2.3300e-02, -2.1507e-01,\n"," 5.5366e-02, 4.4726e-02, -1.0376e-01, -3.2909e-01, 8.6371e-02,\n"," 4.2664e-03, -3.1455e-02, 1.6310e-01, 1.5648e-01, -4.2880e-02,\n"," -1.8474e-02, 1.3106e-01, 1.1363e-01, 2.7173e-02, 1.2660e-01,\n"," -5.9311e-02, -9.8891e-02, -4.2543e-02, -2.3191e-02, 6.0689e-02,\n"," 4.0283e-02, -6.9700e-04, -1.0824e-01, 2.1883e-01, -2.0839e-02,\n"," -1.1635e-01, 1.1721e-01, -4.8828e-02, -2.2544e-02, 1.9957e-01,\n"," 4.6466e-02, 2.6236e-02, -1.5585e-01, 9.1094e-03, -2.1267e-01,\n"," 2.5769e-01, -7.3468e-02, -9.2528e-02, 2.4997e-02, -1.4215e-01,\n"," -1.4417e-01, -3.3305e-01, 1.0021e-01, -5.8869e-02, 7.3044e-02,\n"," 6.0383e-02, -1.3503e-01, 2.7302e-02, 1.8703e-01, -1.2806e-02,\n"," 5.7073e-02, 9.0280e-02, -1.5143e-02, -5.4023e-02, 1.2399e-01,\n"," -8.5139e-02, 6.4925e-02, -5.8222e-02, -7.2756e-02, 8.8541e-02,\n"," 6.0879e-02, -9.2811e-02, 4.9281e-02, -9.5763e-02, -7.7652e-02,\n"," -4.0214e-02, -2.3958e-02, -1.6052e-01, -3.0753e-02, -1.1209e-01,\n"," -7.4997e-03, 5.1728e-02, 1.8273e-02, -7.1002e-02, -7.7520e-02,\n"," 6.3680e-02, 4.9214e-02, -5.5259e-01, -6.9264e-02, -5.8530e-02,\n"," 1.5903e-01, 2.4740e-02, -1.7233e-01, 7.9766e-02, 8.7400e-02,\n"," 1.0187e-01, 2.8480e-02, 1.7869e-02, -2.1188e-01, 1.7296e-01,\n"," 2.1859e-01, -5.0978e-02, -1.9529e-01, 5.9238e-02, 8.6392e-02,\n"," -1.0268e-01, -1.7420e-01, 8.2962e-02, -1.4705e-01, -5.6892e-02,\n"," 1.1193e-01, -1.1189e-02, 8.3530e-02, 1.0402e-02, 2.0608e-02,\n"," 3.3614e-02, -3.3329e-02, -1.1878e-02, -1.5145e-01, -1.0961e-01,\n"," -1.3519e-01, 4.4461e-02, -2.4229e-03, 1.2559e-01, -7.7785e-02,\n"," -7.7474e-02, 8.0846e-02, -1.1829e-01, -9.2990e-02, -1.6285e-01,\n"," 1.4514e-02, -6.7827e-02, 3.9259e-02, -3.3612e-01, 7.6199e-02,\n"," -4.9862e-02, -1.6196e-01, -9.4633e-02, -3.6492e-02, -3.7328e-02,\n"," -4.1201e-03, 9.9998e-03, 7.5270e-02, -5.7569e-02, -4.2786e-02,\n"," 8.6815e-02, 2.8800e-02, 3.1197e-02, 1.0185e-01, 9.4486e-02,\n"," 2.1282e-02, 6.0674e-02, -7.4639e-02, 1.8090e-01, -1.0450e-01,\n"," -2.5522e-01, -5.6027e-02, 1.3828e-01, 1.1350e-01, 2.1448e-02,\n"," -3.9067e-02, -1.3588e-01, -2.9345e-02, -5.5968e-02, -9.8969e-02,\n"," -7.4646e-02, 1.0464e-01, -1.4783e-01, -3.1463e-02, 3.2958e-02,\n"," -1.5104e-01, -6.4073e-02, 2.3301e-01, -5.4972e-02, -3.1530e-02,\n"," 7.5931e-02, -4.6505e-02, 1.4346e-01, 8.4920e-02, -3.2047e-02,\n"," -2.1088e-01, 1.6876e-01, 3.3212e-02, -1.2510e-01, -1.2308e-01,\n"," -1.9802e-02, -3.3015e-02, -2.0518e-01, 4.3430e-02, -3.7013e-02,\n"," 1.2003e-01, -3.7512e-02, 6.9324e-02, -9.7411e-02, -2.1957e-02,\n"," -1.9214e-02, -1.3567e-01, 1.6970e-01, -3.1553e-02, 1.5147e-02,\n"," -5.8446e-02, 2.7342e-02, 6.1681e-02, 1.3681e-02, 2.5004e-01,\n"," -3.5267e-01, 1.6333e-01, -3.8555e-03, -1.3828e-01, -4.7451e-02,\n"," -6.6291e-02, -7.0574e-02, 1.1679e-01, -1.3863e-01, -8.5180e-03,\n"," -1.7836e-03, -7.6153e-02, -1.5434e-02, -8.4037e-02, 9.1066e-02,\n"," -1.2908e-01, -2.0826e-01, 1.4944e-01, -9.1995e-02, -2.6867e-01,\n"," 2.3612e-02, 1.1319e-01, -6.1646e-02, 1.4786e-01, -5.1432e-02,\n"," -1.1507e-01, -3.0265e-02, -1.5270e-01, 5.3833e-02, 9.9415e-02,\n"," -1.3082e-01, 6.2575e-02, -8.8731e-02, -3.9710e-02, -1.2554e-01,\n"," 3.0715e-02, 6.9132e-02, -6.7005e-02, -2.5458e-02, 3.8869e-02,\n"," -1.5923e-01, 2.7743e-02, 2.0169e-02, -4.5735e-02, -9.9385e-02,\n"," -5.1417e-02, 7.3213e-02, 2.4986e-01, -5.2360e-03, -6.5376e-02,\n"," 7.0374e-02, 3.8748e-02, -2.0792e-02, 9.2837e-02, -1.0096e-01,\n"," -3.0281e-02, 7.5467e-02, -8.1408e-02, 5.5916e-02, 1.0426e-01,\n"," -6.8461e-02, -4.2714e-03, -1.5302e-01, 1.4000e-01, 8.2922e-02,\n"," -3.5530e-02, 2.7031e-02, -1.3187e-02, 1.6040e-01, 4.0959e-01,\n"," 6.4632e-04, -3.4900e-02, 4.8370e-04, -4.7273e-02, 1.6765e-01,\n"," -6.1726e-03, 9.9889e-02, 1.9799e-01, 7.6185e-02, 1.1998e-01,\n"," -9.0053e-02, 1.1475e-02, -7.9212e-02, 5.0373e-02, -1.2276e-02,\n"," -1.3934e-01, -2.6125e-01, 2.1754e-02, -1.0877e-01, -1.7861e-02,\n"," -1.6668e-01, -9.2420e-02, -1.3878e-02, -8.8295e-03, -6.3030e-02,\n"," -1.0543e-01, 4.8202e-02, -6.5044e-02, 1.1086e-01, -2.7024e-02,\n"," -8.4654e-02, -5.4931e-02, 5.4255e-02, 6.8870e-02, -5.2517e-02,\n"," -6.2799e-03, 6.2461e-02, 4.3790e-02, -1.5881e-01, 2.1251e-01,\n"," -8.7122e-02, 2.6258e-02, 2.5049e-03, -1.5120e-01, -1.4632e-01,\n"," 4.5138e-03, -9.0443e-03, -2.6588e-02, -1.0465e-01, -4.1162e-02,\n"," 3.7104e-02, -2.6799e-02, 7.3106e-02, 1.5287e-02, -6.9980e-02,\n"," 4.1658e-02, 1.1324e-02, 1.0969e-01, -4.7242e-01, 3.8726e-02,\n"," -5.8977e-02, -1.5502e-01, 1.1687e-01, -9.4594e-02, 8.0425e-02,\n"," 1.4174e-01, -2.3491e-01, -8.6311e-02, 1.7286e-01, 1.2864e-01,\n"," -9.7945e-02, 8.4990e-03, 2.3695e-02, 1.6841e-02, 2.0246e-01,\n"," -3.2752e-02, 2.2906e-02, 2.4480e-01, -1.5979e-01, -1.7827e-02,\n"," 8.6079e-03, 6.5474e-03, -9.0049e-03, -4.3515e-02, -5.6294e-02,\n"," 2.1401e-01, 1.5029e-01, 7.9673e-02, 5.4913e-02, 3.9160e-02,\n"," 1.2875e-01, 3.3008e-02, -5.5979e-02, 1.0740e-01, -3.5599e-02,\n"," 4.3530e-02, 1.3297e-01, -4.8124e-02, 9.8446e-03, 2.2264e-01,\n"," 3.4221e-02, 5.9539e-02, 6.7812e-02, -1.6882e-01, -9.7988e-02,\n"," -9.7874e-02, -6.1450e-02, -6.6836e-02, -7.5918e-02, -3.2872e-01,\n"," 4.8814e-02, 1.1171e-01, 9.3693e-02, -1.3552e-01, -1.2159e-02,\n"," -9.4724e-02, -7.4638e-02, 2.0551e-01, -6.7493e-02, 1.2942e-01,\n"," 8.2681e-02, 9.2897e-02, -1.4535e-01, 4.6495e-02, 1.5323e-02,\n"," 7.4460e-02, -6.0297e-02, 2.4888e-03, -1.1828e-01, -1.9603e-01,\n"," 7.5323e-03, 9.1524e-02, -6.6671e-02, 6.5601e-02, 7.5466e-02,\n"," 1.5524e-02, 1.4577e-02, -6.2519e-03, 4.5152e-02, 1.7786e-02,\n"," 1.4978e-01, -9.3021e-02, 1.3635e-02, 1.1171e-01, 1.6229e-03,\n"," -1.3502e-01, -4.8030e-02, 1.9070e-01, -2.0233e-01, -6.8417e-03,\n"," -2.5035e-01, -1.6159e-01, 6.2476e-03, 1.4825e-01, 1.0560e-01,\n"," -4.0371e-02, 1.9100e-01, 4.2668e-02, 1.3999e-02, 2.9254e-01,\n"," 3.9176e-02, 1.3214e-01, 2.8551e-02, 4.5838e-02, -6.3797e-02,\n"," -9.0351e-02, -2.0034e-01, 3.3103e-02, -2.2779e-01, -6.6089e-02,\n"," -7.3375e-02, 2.8297e-01, 7.9907e-02, -1.0679e-02, -1.0524e-01,\n"," 9.3045e-02, -1.9267e-02, -5.3104e-02, 1.9750e-01, 9.3863e-02,\n"," 3.4340e-02, -1.1681e-01, -9.3470e-02, -7.6323e-02, -1.5271e-02,\n"," 1.0952e-01, 3.0217e-02, -2.1070e-03, -9.3585e-02, 3.0755e-01,\n"," -1.4883e-01, 1.3984e-01, -2.3806e-02, 9.8132e-02, -6.3319e-02,\n"," 2.0151e-01, 7.7652e-02, 1.0479e-01, -5.5277e-02, -5.8733e-02,\n"," -5.0263e-02, 3.6644e-02, 3.9631e-02, -2.1010e-02, -1.1201e-02,\n"," 7.8469e-02, -8.0196e-02, -3.4609e-02, -2.7329e-02, 5.6554e-02,\n"," 2.2118e-02, -5.9166e-02, -1.7367e-01, 2.7787e-02, 1.0721e-01,\n"," 7.8055e-02, -2.9740e-02, -2.4302e-02], grad_fn=)"]},"metadata":{"tags":[]},"execution_count":8}]},{"cell_type":"code","metadata":{"id":"ZT5lmGVK60mJ","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1613206381381,"user_tz":-540,"elapsed":21326,"user":{"displayName":"­권태확","photoUrl":"","userId":"00214940863527742140"}},"outputId":"5cd883ad-e03f-407a-fd7b-051248e955d9"},"source":["text = \"The chicken didn't cross the road because it was\"\n","\n","# Tokenize the input string\n","input = tokenizer.encode(text, return_tensors=\"pt\")\n","\n","# Run the model\n","output = model.generate(input, max_length=20, do_sample=True)\n","\n","# Print the output\n","print('\\n',tokenizer.decode(output[0]))"],"execution_count":9,"outputs":[{"output_type":"stream","text":["Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"],"name":"stderr"},{"output_type":"stream","text":["\n"," The chicken didn't cross the road because it was too late,\" said Mike Ziegler, the\n"],"name":"stdout"}]}]} \ No newline at end of file