diff --git a/docs/source/JAX_for_LLM_pretraining.ipynb b/docs/source/JAX_for_LLM_pretraining.ipynb index 9d1b013..a432823 100644 --- a/docs/source/JAX_for_LLM_pretraining.ipynb +++ b/docs/source/JAX_for_LLM_pretraining.ipynb @@ -42,28 +42,19 @@ "base_uri": "https://localhost:8080/" }, "id": "6zMsOIc7ouCO", - "outputId": "037d56a9-b18f-4504-f80a-3a4fa2945068" + "outputId": "ad486e3b-dd63-405f-d786-79b0b6d60cbd" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m780.7/780.7 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m54.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m289.6/289.6 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m270.5/270.5 kB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.5/62.5 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.3/15.3 MB\u001b[0m \u001b[31m61.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m128.1/128.1 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.4/42.4 kB\u001b[0m \u001b[31m1.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.5/49.5 kB\u001b[0m \u001b[31m1.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25h\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", - "tensorflow 2.15.0 requires ml-dtypes~=0.2.0, but you have ml-dtypes 0.4.0 which is incompatible.\u001b[0m\u001b[31m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m15.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m419.0/419.0 kB\u001b[0m \u001b[31m20.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.3/8.3 MB\u001b[0m \u001b[31m99.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.0/61.0 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/1.2 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r", + "\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━\u001b[0m \u001b[32m0.9/1.2 MB\u001b[0m \u001b[31m26.7 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m21.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m478.8/478.8 kB\u001b[0m \u001b[31m20.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.6/8.6 MB\u001b[0m \u001b[31m110.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.3/65.3 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h" ] } @@ -91,7 +82,7 @@ "base_uri": "https://localhost:8080/" }, "id": "LS9sQEY3n0mB", - "outputId": "9ffcf3a6-20ef-4f80-b006-f5d3c5644a15" + "outputId": "c18a63d0-696e-4c93-f8d5-8c7942045005" }, "outputs": [ { @@ -134,28 +125,28 @@ "base_uri": "https://localhost:8080/" }, "id": "wUjQsgQEmI1N", - "outputId": "e6eff24e-5578-4277-a0f9-24e27bd91ee0" + "outputId": "8a75571a-8339-4f1a-958b-5805aa285bb9" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "--2024-11-01 02:50:38-- https://huggingface.co/datasets/roneneldan/TinyStories/resolve/main/TinyStories-train.txt?download=true\n", - "Resolving huggingface.co (huggingface.co)... 65.8.243.46, 65.8.243.92, 65.8.243.90, ...\n", - "Connecting to huggingface.co (huggingface.co)|65.8.243.46|:443... connected.\n", + "--2025-04-25 01:37:13-- https://huggingface.co/datasets/roneneldan/TinyStories/resolve/main/TinyStories-train.txt?download=true\n", + "Resolving huggingface.co (huggingface.co)... 18.172.134.124, 18.172.134.4, 18.172.134.24, ...\n", + "Connecting to huggingface.co (huggingface.co)|18.172.134.124|:443... connected.\n", "HTTP request sent, awaiting response... 302 Found\n", - "Location: https://cdn-lfs.hf.co/repos/42/7f/427f7497b6c6596c18b46d5a72e61364fcad12aa433c60a0dbd4d344477b9d81/c5cf5e22ff13614e830afbe61a99fbcbe8bcb7dd72252b989fa1117a368d401f?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27TinyStories-train.txt%3B+filename%3D%22TinyStories-train.txt%22%3B&response-content-type=text%2Fplain&Expires=1730688639&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTczMDY4ODYzOX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5oZi5jby9yZXBvcy80Mi83Zi80MjdmNzQ5N2I2YzY1OTZjMThiNDZkNWE3MmU2MTM2NGZjYWQxMmFhNDMzYzYwYTBkYmQ0ZDM0NDQ3N2I5ZDgxL2M1Y2Y1ZTIyZmYxMzYxNGU4MzBhZmJlNjFhOTlmYmNiZThiY2I3ZGQ3MjI1MmI5ODlmYTExMTdhMzY4ZDQwMWY%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qJnJlc3BvbnNlLWNvbnRlbnQtdHlwZT0qIn1dfQ__&Signature=oQHJBcHVix9N1HnNsJSj7KK-BoqdXdl6NRh%7E1ilGx-ROnLrZxKINfonOtva5e5Xf9KQVNl6QQkx5gNw4iMTmS6JRFB%7EcXdTcFjrHSnBxwLRZkMCBKAv3oHhRnJ6I2rV8iBAZTq%7E-caDCLFvBrgT9pcEFakh3-5mSp%7ER7hnNqE5lcE5n7tzXS0l-8tOShDmR5aUCFPStZHfPbyS3MwCAdc2KoqXdqzRf9M4WvXWB78El7WGxse0DrTQFbGGW1kjpvBOqzljH0Qn6WqsiBockhHDbwE1nQmGfxKrbreXenAKdOsUTN9fuRKl-6srhI2xGKFpfu3IGDEN%7Ebmwg8CnwAfQ__&Key-Pair-Id=K3RPWS32NSSJCE [following]\n", - "--2024-11-01 02:50:39-- https://cdn-lfs.hf.co/repos/42/7f/427f7497b6c6596c18b46d5a72e61364fcad12aa433c60a0dbd4d344477b9d81/c5cf5e22ff13614e830afbe61a99fbcbe8bcb7dd72252b989fa1117a368d401f?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27TinyStories-train.txt%3B+filename%3D%22TinyStories-train.txt%22%3B&response-content-type=text%2Fplain&Expires=1730688639&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTczMDY4ODYzOX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5oZi5jby9yZXBvcy80Mi83Zi80MjdmNzQ5N2I2YzY1OTZjMThiNDZkNWE3MmU2MTM2NGZjYWQxMmFhNDMzYzYwYTBkYmQ0ZDM0NDQ3N2I5ZDgxL2M1Y2Y1ZTIyZmYxMzYxNGU4MzBhZmJlNjFhOTlmYmNiZThiY2I3ZGQ3MjI1MmI5ODlmYTExMTdhMzY4ZDQwMWY%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qJnJlc3BvbnNlLWNvbnRlbnQtdHlwZT0qIn1dfQ__&Signature=oQHJBcHVix9N1HnNsJSj7KK-BoqdXdl6NRh%7E1ilGx-ROnLrZxKINfonOtva5e5Xf9KQVNl6QQkx5gNw4iMTmS6JRFB%7EcXdTcFjrHSnBxwLRZkMCBKAv3oHhRnJ6I2rV8iBAZTq%7E-caDCLFvBrgT9pcEFakh3-5mSp%7ER7hnNqE5lcE5n7tzXS0l-8tOShDmR5aUCFPStZHfPbyS3MwCAdc2KoqXdqzRf9M4WvXWB78El7WGxse0DrTQFbGGW1kjpvBOqzljH0Qn6WqsiBockhHDbwE1nQmGfxKrbreXenAKdOsUTN9fuRKl-6srhI2xGKFpfu3IGDEN%7Ebmwg8CnwAfQ__&Key-Pair-Id=K3RPWS32NSSJCE\n", - "Resolving cdn-lfs.hf.co (cdn-lfs.hf.co)... 3.167.152.12, 3.167.152.119, 3.167.152.37, ...\n", - "Connecting to cdn-lfs.hf.co (cdn-lfs.hf.co)|3.167.152.12|:443... connected.\n", + "Location: https://cdn-lfs.hf.co/repos/42/7f/427f7497b6c6596c18b46d5a72e61364fcad12aa433c60a0dbd4d344477b9d81/c5cf5e22ff13614e830afbe61a99fbcbe8bcb7dd72252b989fa1117a368d401f?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27TinyStories-train.txt%3B+filename%3D%22TinyStories-train.txt%22%3B&response-content-type=text%2Fplain&Expires=1745548633&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTc0NTU0ODYzM319LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5oZi5jby9yZXBvcy80Mi83Zi80MjdmNzQ5N2I2YzY1OTZjMThiNDZkNWE3MmU2MTM2NGZjYWQxMmFhNDMzYzYwYTBkYmQ0ZDM0NDQ3N2I5ZDgxL2M1Y2Y1ZTIyZmYxMzYxNGU4MzBhZmJlNjFhOTlmYmNiZThiY2I3ZGQ3MjI1MmI5ODlmYTExMTdhMzY4ZDQwMWY%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qJnJlc3BvbnNlLWNvbnRlbnQtdHlwZT0qIn1dfQ__&Signature=o3wAd1AJqh8HJqOoCSsN2UhG0CD18l4DvfzfbXQYuGu1ULd41OH67qYah6Gqa8UoiOR-vY2mL68PRKmR5xzN86-u0A-ONmHjeXfKuQj3JtAD2jJQ9Y1IvItldi8bW6yfpKMqtz8VKQ5iU%7EWsjkopTP%7EuLX%7EqbJleJ%7E2QoIeMfHQGSA-5ijnrTlJdjcsrlP-owPmmZ0xS8cWPFYLIFrL4Wi3JuddcN1AZDY9XraKobFVUrzzCgJBF5xgmRfGBejZbWtmm6VBhViB1m1CSoPSji5mrRlr1LclBFBmAcrkQ2QuPWjgYGBT7ONwmVSI0kjTO09z0%7E8mneFC3vQYJIaLItw__&Key-Pair-Id=K3RPWS32NSSJCE [following]\n", + "--2025-04-25 01:37:13-- https://cdn-lfs.hf.co/repos/42/7f/427f7497b6c6596c18b46d5a72e61364fcad12aa433c60a0dbd4d344477b9d81/c5cf5e22ff13614e830afbe61a99fbcbe8bcb7dd72252b989fa1117a368d401f?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27TinyStories-train.txt%3B+filename%3D%22TinyStories-train.txt%22%3B&response-content-type=text%2Fplain&Expires=1745548633&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTc0NTU0ODYzM319LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5oZi5jby9yZXBvcy80Mi83Zi80MjdmNzQ5N2I2YzY1OTZjMThiNDZkNWE3MmU2MTM2NGZjYWQxMmFhNDMzYzYwYTBkYmQ0ZDM0NDQ3N2I5ZDgxL2M1Y2Y1ZTIyZmYxMzYxNGU4MzBhZmJlNjFhOTlmYmNiZThiY2I3ZGQ3MjI1MmI5ODlmYTExMTdhMzY4ZDQwMWY%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qJnJlc3BvbnNlLWNvbnRlbnQtdHlwZT0qIn1dfQ__&Signature=o3wAd1AJqh8HJqOoCSsN2UhG0CD18l4DvfzfbXQYuGu1ULd41OH67qYah6Gqa8UoiOR-vY2mL68PRKmR5xzN86-u0A-ONmHjeXfKuQj3JtAD2jJQ9Y1IvItldi8bW6yfpKMqtz8VKQ5iU%7EWsjkopTP%7EuLX%7EqbJleJ%7E2QoIeMfHQGSA-5ijnrTlJdjcsrlP-owPmmZ0xS8cWPFYLIFrL4Wi3JuddcN1AZDY9XraKobFVUrzzCgJBF5xgmRfGBejZbWtmm6VBhViB1m1CSoPSji5mrRlr1LclBFBmAcrkQ2QuPWjgYGBT7ONwmVSI0kjTO09z0%7E8mneFC3vQYJIaLItw__&Key-Pair-Id=K3RPWS32NSSJCE\n", + "Resolving cdn-lfs.hf.co (cdn-lfs.hf.co)... 3.167.152.106, 3.167.152.37, 3.167.152.12, ...\n", + "Connecting to cdn-lfs.hf.co (cdn-lfs.hf.co)|3.167.152.106|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 1924281556 (1.8G) [text/plain]\n", "Saving to: ‘TinyStories-train.txt’\n", "\n", - "TinyStories-train.t 100%[===================>] 1.79G 38.1MB/s in 45s \n", + "TinyStories-train.t 100%[===================>] 1.79G 259MB/s in 7.3s \n", "\n", - "2024-11-01 02:51:24 (40.7 MB/s) - ‘TinyStories-train.txt’ saved [1924281556/1924281556]\n", + "2025-04-25 01:37:20 (251 MB/s) - ‘TinyStories-train.txt’ saved [1924281556/1924281556]\n", "\n" ] } @@ -446,48 +437,31 @@ " outputs = self.output_layer(x)\n", " return outputs\n", "\n", - " # Text generation.\n", - " def generate_text(self, max_tokens: int, start_tokens: [int], top_k=10):\n", - " # Sample the next token from a probability distribution based on\n", - " # `logits` and `tok_k` (top-k) sampling strategy.\n", - " def sample_from(logits):\n", - " logits, indices = jax.lax.top_k(logits, k=top_k)\n", - " # Convert logits to probabilities (using `flax.nnx.softmax`).\n", - " logits = nnx.softmax(logits)\n", - " return jax.random.choice(jax.random.PRNGKey(0), indices, p=logits)\n", - "\n", - " # Generate text one token at a time until the maximum token limit is reached (`maxlen`).\n", - " def generate_step(start_tokens):\n", - " pad_len = maxlen - len(start_tokens)\n", - " # Index of the last token in the current sequence.\n", - " sample_index = len(start_tokens) - 1\n", - " # If the input is longer than `maxlen`, then truncate it.\n", - " if pad_len < 0:\n", - " x = jnp.array(start_tokens[:maxlen])\n", - " sample_index = maxlen - 1\n", - " # If the input is shorter than `maxlen`, then pad it (`pad_len`).\n", - " elif pad_len > 0:\n", - " x = jnp.array(start_tokens + [0] * pad_len)\n", - " else:\n", - " x = jnp.array(start_tokens)\n", - "\n", - " # Add a batch dimension.\n", - " x = x[None, :]\n", - " logits = self(x)\n", - " next_token = sample_from(logits[0][sample_index])\n", - " return next_token\n", - "\n", - " # Store generated tokens.\n", + " @nnx.jit\n", + " def sample_from(self, logits):\n", + " logits, indices = jax.lax.top_k(logits, k=top_k)\n", + " logits = nnx.softmax(logits)\n", + " return jax.random.choice(jax.random.PRNGKey(0), indices, p=logits)\n", + "\n", + " @nnx.jit\n", + " def generate_step(self, padded_tokens, sample_index):\n", + " logits = self(padded_tokens)\n", + " next_token = self.sample_from(logits[0][sample_index])\n", + " return next_token\n", + "\n", + " def generate_text(self, max_tokens, start_tokens):\n", " generated = []\n", - " # Generate tokens until the end-of-text token is encountered or the maximum token limit is reached.\n", - " for _ in range(max_tokens):\n", - " next_token = generate_step(start_tokens + generated)\n", - " # Truncate whatever is after '<|endoftext|>' (stop word)\n", + " print(tokenizer.decode(start_tokens), flush=True, end='')\n", + " for i in range(max_tokens):\n", + " sample_index = len(start_tokens) + len(generated) - 1\n", + "\n", + " padded_tokens = jnp.array((start_tokens + generated + [0] * (maxlen - len(start_tokens) - len(generated))))[None, :]\n", + " next_token = int(self.generate_step(padded_tokens, sample_index))\n", " if next_token == tokenizer.encode('<|endoftext|>', allowed_special={'<|endoftext|>'})[0]:\n", - " # Stop text generation if the end-of-text token is encountered.\n", " break\n", - " generated.append(int(next_token))\n", - " # Decode the generated token IDs into text.\n", + " generated.append(next_token)\n", + " # decode and print next_token\n", + " print(tokenizer.decode([next_token]), flush=True, end='')\n", " return tokenizer.decode(start_tokens + generated)\n", "\n", "# Creates the miniGPT model with 4 transformer blocks.\n", @@ -519,7 +493,8 @@ "num_heads = 8\n", "feed_forward_dim = 256\n", "batch_size = 256 # You can set a bigger batch size if you use Kaggle's Cloud TPU.\n", - "num_epochs = 1" + "num_epochs = 1\n", + "top_k = 10" ] }, { @@ -639,7 +614,7 @@ "base_uri": "https://localhost:8080/" }, "id": "Ysl6CsfENeJN", - "outputId": "5dd06dca-f030-4927-a9b6-35d412da535c" + "outputId": "b713cc63-a4ff-4fea-96ea-c234348e0ea4" }, "outputs": [ { @@ -647,332 +622,359 @@ "output_type": "stream", "text": [ "Initial generated text:\n", - "Once upon a time Christina Raven Liqu Everyday seaw Spl digit mini Hungarian wasteful USC recurrent brawl towers summAvailability manualsidsAvailability Jord staleEarlier 303 Latter soakinginated pierced acquaint propaganda differentlyBesides Splambling Significant processing locals FoundingFlickrverbalSquaresth pixels CON repetitivebass%; dartsKN ushered sim wasteful Qi510 174 (_ Hillaryall hopeddalePref recurrentbassoves AOL ushered Hunt manuals NietzscheidsBY Equ souls correctedresaKN ghamblinguador contest cornerback bannedKN realizedSix summlargest gh fastest req influences cursingosureelse delighted wrecked donors codsedentiallyindaletteogenicAI summ wasteful USCesm shaped Garrett resistance grandchildren souls babyStatementambling fastestirin AWSiden groundedKen%; aboarddogs seaw Sultan Sachs Sonic ArchivesINE darts belts asylumei simette expands targetintergroupon Graveyard Graveyard398Jordan 66 medication Leadership 174?: seaw manuals summ asylumrw slice manualsiries Prometheus� Seat correctedINE denomination summ vastlyKNKN belts?: contest PamelaidiumKN themHI seawKN minions summ squadKN Joker sacredamblingKNuckyKNette 69 Xan 69ourse notificationuku Sitting cosmeticakesGro McAuliffeilles Graveyard differe <-Jordan Archives 180 Puppet cabinetodcast spir305 bannedambling 66 medicationbass victory relatingakespe Rover GarrettPrefppo sim recurrent manualsidsrg eveningsossus asylum Puppet hydra SultanProxy 66 chew Jokeranswer%;Loc Australian awaidiumdale landed Luahangはambling SuddenlyKN victory victory victory victory\n", + "Once upon a timeaciaGender gearuser Analysisval {} Bruce Lauren helic Lauren Bruce againstliterally SQU retire Path {}valascript northwest {} Bruceuit Pathascript northwestdrops freelyvic996 curated hysteria survivor {}sclaxteradvert Sitting qualifiers snack {} scenariovalameron {} Path {}Nick VeganExcept peasantascript Whites retire {} retire {} Analysisrest {} Mine psychedelic flankForgeModLoader Path Bravo {} inflic {} strutConnector psychedelic beyond Beforeocker interesting Dani {}sclaxter retire {}Nick sorrow Typesrest interestingUV FSyrus resorts {} Dani {} perished {} retire interesting sorrow reversibleurned {} Womanlast 118 reass gentlestudyManager {} retire {} verb Captain forbid Bruce {} Analysis ox {} inexplicable tumor psychedelic {} serverpelrest Sky {} cropDisclaimeruti Nortonocated twins Path {} psychedeliccre motionsundrum {} northwestroid variable {} Whites {} dancers iPod {} {} verb retire {} Fred Noble {}ampionscre lineman servesShould decision1024� serveraez {} retire interesting Tangrest Carly juice,. allowsmodulerest Antarsumameroncre Flesh --> northwestroidENN {} Gustav rolledMuch challengundrum {}val retire {}scl less {} perished Brigham Analysis developersSomething hiding {}scl Houthval {} northwest appease miles {} escalationManager {} northwest {} {} Cube psychedelic {} inflic {} retire {} Whites dancers {}scl FS lore appease Din {} Whites abnorm[] {} {}scl FS appease dangling Bruce abnormcre97 psychedeliccre!!!\n", "\n", - "Step 200, Loss: 4.541538715362549, Elapsed Time: 119.14 seconds\n", + "Step 200, Loss: 4.653054714202881, Elapsed Time: 100.71 seconds\n", "Generated text:\n", - "Once upon a time was a time was was was was very little girl, day was so happy, her little girl, her little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little!!!!\n", + "Once upon a time, there a little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little!!!!\n", "\n", - "Step 400, Loss: 2.8348119258880615, Elapsed Time: 103.58 seconds\n", + "Step 400, Loss: 3.0780816078186035, Elapsed Time: 59.71 seconds\n", "Generated text:\n", - "Once upon a time there was a small cat named Jack. Tim was a little girl who lived in the world and wanted to explore the forest. The dog, he had a new toy car and his mommy and daddy. He wanted to be so much. He went to help the ball. He said to be careful to go and said, \"What's wrong!\"\n", - "After they found the other kids said, Timmy was so excited and he could see his friend. Timmy said, \"Let's play in the other animals, but I want to help.\"\n", - "Timmy smiled and they played with a while. Timmy said, \"It's not listen to play. \"Thank you. \"You are happy.\"\n", + "Once upon a time there was a little girl named Lily. She loved to play outside and play with her mommy was very much fun. She loved to play with her mommy and her mommy's mommy's mommy said, but she went to her mommy's mommy's mommy's mommy said, \"I'm going to play with her mommy.\n", + "Lily said, \"I'm going to the park. She said, \"I'm going to play with her mommy said, \"I'm going to play with you want to play with you can't want to play with her mommy and said, \"I'm so happy to play with her mommy and said, \"I'm sorry, \"I'm sorry, \"I'm sorry, I can't worry, \"I'm sorry, I can't want to be a good.\"\n", "\n", "\n", - "Step 600, Loss: 2.3290421962738037, Elapsed Time: 69.78 seconds\n", + "Step 600, Loss: 2.4993953704833984, Elapsed Time: 31.43 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She was very curious and loved to play outside in the garden. One day, she was playing with her toys and saw a big box with a shiny red ball. The ball was shiny and wanted to climb it, but she didn't want to be too far. \n", - "\"Hi, I can't get my toy!\" said, but Lily said, \"No, you are not yours.\" \n", - "Lily's mom smiled and said, \"You can help me go to me!\"\n", - "\"I want to get some candy and it,\" she said. \"That's a big dog.\"\n", - "The cat said, \"No, we need to go home. I'm going to find some.\" \n", - "Her mom said, \"Okay, you will get to play with me.\"\n", - "Lily smiled and said, \"I want to get my favorite toy first!\" \n", - "Her mom said, \"Okay, but they don't know. They both thought it is so much fun to go on their mom and have fun. They played together, but they did not wait to be friends and they had fun together. \n", - "\n", - "\n", - "Step 800, Loss: 2.049470901489258, Elapsed Time: 84.69 seconds\n", + "Once upon a time, there was a little girl named Lily. She loved to play with her toys and play with her toys. One day, she went to the park with her mommy and daddy.\n", + "\"Let's go to the park!\" Lily asked.\n", + "\"Let's go to the park,\" her mommy said.\n", + "\"No, we can't have to go to the park,\" said.\n", + "\"Sure, Lily.\n", + "\"Sure, I want to play with you!\" Lily said.\n", + "\"Yes, I'm going to play with you!\" Lily said.\n", + "\"Yes, I want to play with you!\" Lily said.\n", + "Lily and Lily went to the park. She saw a big dog. She was so happy and had a lot of fun.\n", + "\"Wow, Lily, Lily, Lily, Lily, Lily!\" she said.\n", + "\"I'm sorry, Lily. \"I'm sorry, Lily. I'm sorry, Lily. I can't have a good friend.\"\n", + "Lily and Lily said, \"Yes, Lily. I can't like to play with you.\"\n", + "Lily and Lily and Lily and Lily went to play with her mommy. They played together and Lily played together. They played together and played!!!!\n", + "\n", + "Step 800, Loss: 2.1457183361053467, Elapsed Time: 32.42 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lucy was playing in the garden. She loved to run and play outside and she had lots of fun. She would always look at all day and her friends in the garden.\n", - "One day, she was playing outside when she heard a strange noise coming from a loud noise. It was coming from a dark and it started to shake. It was a big dog, so scared.\n", - "Her mom came into the room and said, \"That's not safe!\"\n", - "The dog smiled and said, \"Don't worry, I will find my friend.\" Lucy and the dog started to walk in the grass, but soon she heard the sound coming from the garden, and she heard a noise. The dog ran back inside, but the door opened and saw that the shadow had a hole in a hole. Lucy and the dog became friends, and they played together all the things and they never found a beautiful spot to get home.\n", + "Once upon a time, there was a little girl named Lily. She loved to play with her toys and her toys. One day, she went to the park with her mommy and dad. She saw a big dog named Max. Lily was scared and ran to Max.\n", + "\"Hi, Lily! What's wrong?\" asked Lily.\n", + "\"I'm sorry, but I saw a big dog. I found a big dog. I found a big dog. I found a big dog and Lily said, \"Don't worry, I will help you.\"\n", + "Lily was happy and said, \"Thank you, Max. You are a good friend.\"\n", + "Lily felt happy and said, \"Thank you, Lily. You are a good friend.\"\n", "\n", "\n", - "Step 1000, Loss: 1.8924535512924194, Elapsed Time: 76.64 seconds\n", + "Step 1000, Loss: 1.9495660066604614, Elapsed Time: 31.16 seconds\n", "Generated text:\n", - "Once upon a time, there was a girl named Amy. Amy loved to go outside and play with her friends. One day, Amy saw a big tree and asked her friends, \"Can I have some best friends?\" The tree was so big that it made a special tree to get in the tree. \n", - "Amy went to the tree and asked, \"Can I go on my tree?\" The tree replied, \"Sure, let's play together!\" \n", - "So, Amy and the tree were very excited. Amy started to play, but it was too heavy for the tree. She started to climb and forth, but it was so far away. Amy's mom saw how fast it was. She asked, \"Why do you think this, but I want to get to be my friend?\" Amy said, \"No, it's too late, so we can't play together!\" Amy's friend started to fight and said, \"I'll be friends. Let's play together!\" Amy and Amy laughed, and laughed together, and they became best friends and they had a fun day together.\n", + "Once upon a time, there was a little girl named Lily. She loved to play with her toys and her toys. One day, she found a big box in the box. She was very happy and wanted to show her mom her mom.\n", + "Lily was very happy. She said, \"Mom, can I have a letter?\"\n", + "Her mom said, \"Sure, but you can't find a letter. It's a letter. It's a letter.\" Lily was so happy and thanked her mom.\n", + "After they finished, Lily's mom said, \"Thank you, Lily. It's a letter.\" Lily was so happy to see her mom. She said, \"Thank you, Lily. I love you, my letter.\"\n", + "Lily was happy to have a new letter. She said, \"Thank you, Lily. I love you, my letter!\"\n", "\n", "\n", - "Step 1200, Loss: 1.7993096113204956, Elapsed Time: 80.74 seconds\n", + "Step 1200, Loss: 1.8355252742767334, Elapsed Time: 31.50 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Amy. She loved to go outside and play in the sunshine. One day, she went outside to play with her friends. They went on a sunny day and found a shiny ball. \n", - "Amy wanted to see the ball and it started to fly it. But then, a little girl came to her friend. Her friend saw her and asked if she could join the bird. The bird said yes and Lily took it to her house.\n", - "Amy was very happy and she started to sing and dance with her friend. They sang together and danced until the sun came up. They laughed and danced together, laughing and having fun. They played together all day and the day long.\n", + "Once upon a time, there was a little girl named Lily. She loved to play with her toys and play with her toys. One day, she found a big box in her room. She was so happy and wanted to play with it.\n", + "Lily's mom said, \"Lily, you can play with your toys. It's a toy car.\" Lily was so happy and said, \"Thank you, Lily. I am so happy to have her toy car.\"\n", + "Lily was happy and said, \"Thank you, Lily. I love you, my car is my car.\"\n", + "Lily was happy and said, \"Thank you, Lily. I love you, Lily. I love you too!\"\n", "\n", "\n", - "Step 1400, Loss: 1.7445931434631348, Elapsed Time: 69.74 seconds\n", + "Step 1400, Loss: 1.7667107582092285, Elapsed Time: 31.08 seconds\n", "Generated text:\n", - "Once upon a time, there was a girl named Sue. She was only three years old. Sue loved to explore. One day, she saw a little bird sitting on a branch. Sue was scared. She ran up to her mom and said, \"Mom, what's that noise?\" Her mom smiled and said, \"It's okay, Sue. We don't have any more fun!\"\n", - "Sue and her mom looked around the house and found a small bird that made her feel better. It was a little bird. Sue and her mom went outside and played in the sun. They had a great time playing and laughing.\n", - "At the end of the day, Sue said, \"Thank you, Mommy! You're welcome! You are very lucky to be a friend.\" Her mom smiled and said, \"Thank you, mommy. I love your new friend!\"\n", + "Once upon a time, there was a little girl named Lily. She loved to play outside in the sunshine. One day, she saw a big red ball on the ground. She wanted to play with it, but her mom said no.\n", + "Lily went to the park and saw a big tree. She wanted to climb it. She climbed up and down the tree. She climbed up and down. She climbed up the tree and climbed up. She felt the wind on her face. She felt so happy and free.\n", + "But then, she heard a loud noise. It was coming from the tree. She was scared and ran away. She ran back to her mom and told her to go back to the tree. Her mom said she had to go to the tree and play on the swings. She was so happy that she had found a new friend. She was so happy to have found her new friend.\n", "\n", "\n", - "Step 1600, Loss: 1.6877646446228027, Elapsed Time: 74.16 seconds\n", + "Step 1600, Loss: 1.7037931680679321, Elapsed Time: 31.64 seconds\n", "Generated text:\n", - "Once upon a time, a boy named Jack was walking down the street. He was feeling very scared. His mom told him that he had to go outside and get to play. \n", - "One day, Jack noticed a little boy playing in the park. He wanted to play with the equipment too, but he was scared. \n", - "Jack ran over and asked, \"Why did you get me?\" His mom said, \"I don't want to be scared. I'm sorry I can't have the equipment to take the equipment to play.\" \n", - "Jack thought about the equipment he would make it even more fun with the equipment. So, he started to build the equipment. When he was finished, Jack felt a bit better. \n", - "The equipment was a way to the playground, but he knew that his mom was not alone, she had to help him get better. She asked him to stay in the playground, but Jack was still happy to have the equipment to stay, but Jack couldn't.\n", - "Jack learned that being kind was important to help and not be afraid of being ignorant and selfish. He was still happy, but he was still very selfish and always asked the equipment for help.\n", + "Once upon a time, there was a little girl named Lily. She loved to play with her toys and play with her toys. One day, she went to the park with her mommy and daddy. She saw a big dog and wanted to pet it. She asked her mommy, \"Can I pet the dog, please?\" Her mommy said, \"No, you can't. I want to pet the dog.\"\n", + "Lily was sad and didn't want to be sad. She wanted to pet the dog, but she was not happy. She said, \"No, you have to be naughty and not to play with the dog. You have to be kind and gentle. You have to be kind and gentle and gentle.\"\n", + "The dog was happy and said, \"Thank you, Lily. You are a good dog. I'm sorry for you. I'm sorry for the dog.\"\n", + "Lily felt bad and said, \"I'm sorry, but I didn't mean to hurt you. I'm sorry for the dog. I didn't mean to hurt you. I'm sorry for the dog. I was mean to be rude and not to hurt you. I'm sorry for you. I'm sorry for you. I'm sorry to forgive you!!!!\n", "\n", - "\n", - "Step 1800, Loss: 1.6456927061080933, Elapsed Time: 85.76 seconds\n", + "Step 1800, Loss: 1.661496639251709, Elapsed Time: 32.36 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Mia. Mia loved to play outside in the park. One day, Mia found a shiny coin in the park. She wanted it for her birthday party, but it was too expensive. Mia thought it would not buy the coin for her birthday.\n", - "So, Mia decided to ask her mom for the coin to buy the coin. The coin looked for a long time and said yes. Mia put the coin back in the park, but it was gone. Mia was very sad.\n", - "So, Mia asked her mom if she could have a special day. Her mom said yes and they both went home with the coin. Mia was so happy! She said yes, but the coin was gone forever.\n", + "Once upon a time, there was a little girl named Lucy. She was very excited to go on a tour with her mommy and daddy.\n", + "When they arrived at the park, Lucy saw a big tree with lots of leaves. She wanted to climb it and see what was on the tree.\n", + "Lucy climbed the tree and climbed the tree. She climbed up the tree and climbed the tree. She climbed higher and higher until she reached the top.\n", + "When Lucy reached the top, she saw a big, scary dog. The dog was barking loudly and Lucy was scared. She ran away, but the dog was too fast.\n", + "Lucy was scared and ran away. She tried to get back, but she couldn't. She was stuck in the tree, but she was too scared. She tried to get back, but the dog was too fast.\n", + "The dog was too fast and Lucy was scared. She tried to run away, but the dog was too fast. The dog was too fast and Lucy was too scared to get back. She was safe and happy.\n", "\n", "\n", - "Step 2000, Loss: 1.6211789846420288, Elapsed Time: 69.42 seconds\n", + "Step 2000, Loss: 1.635001301765442, Elapsed Time: 32.21 seconds\n", "Generated text:\n", - "Once upon a time, there was a small boy named Tim. Tim was a small boy named Timmy. Timmy loved to play with his toy car and he loved to make noises with it.\n", - "One day, Timmy and his toy car went to the park with his toy car. Timmy's car got stuck in the car and it crashed. Timmy's car was too fast and it hit a tree with the wheel.\n", - "Timmy was sad because he loved his car so much. He knew he could help make his toy car better. He said, \"I will make you happy! Let's play together and make a new car.\" And Timmy and his toy car had fun together and they became good friends.\n", + "Once upon a time, there was a little girl named Lucy. She was three years old and loved to play with her toys. One day, Lucy went to the park with her mom. She saw a big tree with lots of leaves. Lucy wanted to climb it.\n", + "Lucy asked her mom, \"Mommy, can I climb the tree?\" Her mom said, \"Yes, but you can climb the tree. It's too high up.\" Lucy was so excited. She climbed up the tree and climbed up. She climbed higher and higher until she reached the top.\n", + "When she got to the top, she saw a big tree with a big tree. She climbed up the tree and climbed up. She climbed up the tree and climbed up. She climbed up the tree and climbed up. She climbed higher and higher, higher and higher until she was almost up.\n", + "When she was done, Lucy was so happy. She had a great adventure and was happy to have helped her mom.\n", "\n", "\n", - "Step 2200, Loss: 1.578463077545166, Elapsed Time: 68.18 seconds\n", + "Step 2200, Loss: 1.5896366834640503, Elapsed Time: 32.09 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lucy. She was only three years old and loved to play with her toys. One day, she found a special treasure, a big, scary toy that was very big and she was so excited to take it home. \n", - "But, one day, Lucy found a shiny jewel that she was very happy to find it. She opened the door and saw a beautiful necklace in the corner of the room. She wanted to see what it was like. She asked her friend, the necklace, and the necklace said, \"Can I take it home now?\" \n", - "Lucy smiled and said, \"I can borrow it with a special diamond. It's very special because it belongs to someone else.\"\n", - "The diamond was so happy to see Lucy and said, \"Thank you, Lucy. You are a great friend!\"\n", + "Once upon a time there was a little girl named Lucy. She was very happy and loved to play with her toys. One day, she found a big box in the box. She was so excited to open it.\n", + "Lucy opened the box and found a big box. Inside the box was a toy car. She was so happy and played with it all day long.\n", + "But then, Lucy accidentally knocked over the box. She was very sad. She wanted to see what was inside the box.\n", + "Lucy's mom saw her and said, \"Lucy, you can't open the box. It's a toy car.\" Lucy was so happy and said, \"Thank you, Mom. I'm so glad I could help.\"\n", "\n", "\n", - "Step 2400, Loss: 1.5326989889144897, Elapsed Time: 73.51 seconds\n", + "Step 2400, Loss: 1.5428193807601929, Elapsed Time: 31.43 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Timmy. Timmy loved to play outside in the sun. One day, Timmy saw a beautiful flower. It was so pretty!\n", - "Timmy asked his mom, \"What is this flower?\"\n", - "His mom replied, \"It's a flower. Do you want to play with it?\"\n", - "Timmy nodded his head and said, \"Yes, please.\"\n", - "Timmy was so happy and he played with the flower. They laughed and played until the sun went down.\n", - "Suddenly, they heard a loud noise. It was a little kitten. The kitten was scared and didn't know what to do. Timmy's mom told him it was a bad ending.\n", + "Once upon a time, there was a little girl named Sarah. She was three years old and loved to play outside. One day, Sarah went to the park to play. She saw a big slide and wanted to go on it. She climbed up and slid down the slide. She was so happy to see the slide.\n", + "But then, Sarah saw a big dog. She was scared and ran away. She ran to her mom and said, \"Mommy, look! It's so big!\" Her mom smiled and said, \"Yes, it's okay. We can go on the slide again.\"\n", + "Sarah was so happy and said, \"Thank you, Mommy! I'm so glad I could go to the slide again!\" Her mom smiled and said, \"Me too! Let's go!\"\n", "\n", "\n", - "Step 2600, Loss: 1.5533288717269897, Elapsed Time: 70.30 seconds\n", + "Step 2600, Loss: 1.5633305311203003, Elapsed Time: 31.28 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She was very happy because she had a pretty blouse. One day, Lily was playing with her friend Jack came over to play. She asked, \"Can I play with you?\" \n", - "Jack said, \"Sure, let's play with your friends!\" Lily was excited and said, \"Okay, let's play!\" They ran around the room and played with their new friends. \n", - "As they were playing, Lily accidentally knocked over the blouse and it fell off the blouse. Jack said, \"Ouch!\" \n", - "Lily felt embarrassed and said, \"It's okay, Lily. I'm sorry. I didn't mean to break the blouse. Let's play with it again!\" \n", - "They both started to play and soon the blouse was over. They played together and had a lot of fun. Lily felt so happy that she didn't make her friend laugh again. She realized that sometimes things don't seem too bad and we should be different, but they still have a solution.\n", + "Once upon a time, there was a little girl named Lucy. She was very curious and loved to explore. One day, she found a big box in the attic. She was curious and wanted to see what was inside.\n", + "Lucy opened the box and saw many things inside. She was curious and wanted to see what was inside. She opened the box and saw many things inside. She saw a big, scary monster.\n", + "Lucy was scared and ran away. She hid behind the box and waited for the monster to come back. The monster was hiding behind the box and the box was gone. Lucy was sad and scared. She wanted to go back to the box and play with the monster.\n", + "The monster was very angry and scared. It ran away from the box and Lucy never came back. The end.\n", "\n", "\n", - "Step 2800, Loss: 1.5323652029037476, Elapsed Time: 80.58 seconds\n", + "Step 2800, Loss: 1.541506052017212, Elapsed Time: 31.95 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Tim. Tim was a good boy. He liked to play with his toys and his dog, Max. One day, Tim went to the park with his mom.\n", - "\"Hi, Tim! I want to play with you. Do you want to play with me?\" Tim asked.\n", - "\"Yes, Tim. Let's play together!\" his mom said.\n", - "Tim and Max played together all day. They laughed and had lots of fun. Tim and Max were good friends.\n", - "\"Can we play with our friends?\" Max asked.\n", - "\"Yes, but we can play together,\" his mom said.\n", - "Tim and Max played together. They laughed and had fun. Tim and Max were happy to play with them. They were best friends.\n", + "Once upon a time, there was a little girl named Lucy. She was very happy and loved to play with her friends. One day, Lucy's mom said, \"Lucy, you have to go to the park today. It's time for lunch.\" Lucy was sad and didn't want to go to the park.\n", + "But then, a big dog came and started to play. Lucy's mom saw the dog and said, \"Don't worry, Lucy. We can go to the park and play.\" Lucy was so happy and ran to the dog. She ran and ran, but the dog was too fast. Lucy was scared and ran away.\n", + "The next day, Lucy and her mom went back to the park. They saw a big slide and ran to it. Lucy was so happy and said, \"Thank you, mom! You're the best!\" The dog smiled and said, \"You're welcome, Lucy. I'm glad you're safe.\" Lucy smiled and said, \"I'm glad you're safe.\"\n", "\n", "\n", - "Step 3000, Loss: 1.5298235416412354, Elapsed Time: 70.76 seconds\n", + "Step 3000, Loss: 1.538221001625061, Elapsed Time: 31.71 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She was three years old and loved to explore the world around her. One day, Lily found a shiny jewel on the ground. She picked it up and looked inside the jewel. She picked it up and examined it. She showed it to her mom and they were so proud of her. \n", - "After she was finished playing, Lily went to bed. She had so much fun that she didn't realize it was her mom had made a mistake. Her mom explained that sometimes it's important to be responsible and to listen to others. Lily was very careful and listened to her mom's advice.\n", - "From that day on, Lily was careful and she never forgot about the jewel. She always made sure to be careful when she played. And always remember to always be careful when things happened.\n", + "Once upon a time, there was a little girl named Lucy. She was three years old and she loved to play with her toys. One day, Lucy's mommy said, \"Lucy, I want to play with your toys.\" Lucy was very excited and said, \"Yes, please!\"\n", + "So, Lucy and her mommy went to the store to buy some candy. Lucy was so happy and said, \"Thank you, Mommy!\" Her mommy said, \"You're welcome, Lucy. I'm glad you like candy.\" Lucy smiled and said, \"I'm glad you like it.\"\n", + "Lucy's mommy said, \"That's a great idea, Lucy. Let's go home and have some fun together.\" Lucy was so happy and said, \"Yay! I love playing with my toys!\"\n", "\n", "\n", - "Step 3200, Loss: 1.471197485923767, Elapsed Time: 72.75 seconds\n", + "Step 3200, Loss: 1.4795030355453491, Elapsed Time: 31.69 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Tim. Tim was a curious little boy who loved to play. One day, he found a toy car in the garden. It was red and shiny. Tim was very happy and excited.\n", - "As Tim went outside to play, he saw a little bird in the grass. The bird had a red car. Tim wanted to help the bird. He said, \"Hello, little bird. Can you help me lift my toy?\" The bird looked at Tim and said, \"Yes, please! It is so nice to be careful. You are smart and smart.\"\n", - "Tim was very happy to help the bird. He thanked the bird and the bird flew away. Tim learned that helping others can make us feel good and smart. He learned that helping others can make us happy too.\n", + "Once upon a time, there was a little girl named Lucy. She was three years old and loved to play with her toys. One day, Lucy's mom said, \"Lucy, let's go for a walk!\" Lucy was so excited. She ran to the park and saw a big, shiny rock. She picked it up and showed it to her mom.\n", + "\"Look, Mommy! I found a rock!\" said Lucy. \"It's so pretty!\"\n", + "Her mom smiled and said, \"That's a great rock, Lucy. It's a pretty rock. It's a pretty rock.\"\n", + "Lucy put the rock on the rock and rocked back and forth. She felt happy and proud. She had found a new rock and showed it to her mom.\n", "\n", "\n", - "Step 3400, Loss: 1.5036591291427612, Elapsed Time: 71.77 seconds\n", + "Step 3400, Loss: 1.510359764099121, Elapsed Time: 31.32 seconds\n", "Generated text:\n", - "Once upon a time there was a boy named Tim. He loved to go outside and play. One day, he saw a big tree with lots of branches. He wanted to climb it, but his mom said no.\n", - "Tim tried to climb the tree, but it was too high. He was getting higher and higher until he could see what was on the top. Then he heard a loud noise coming from outside. He peeked out from the ground and saw a small bird flying above.\n", - "Tim was so happy to be able to reach the top. He jumped and jumped up and down the tree with excitement. When he was finally caught the bird, it flew down to the tree. Tim was so proud of his success.\n", - "He thanked the bird and continued climbing. He was very happy that he had climbed the tree, even though he couldn't get down from the tree. From then on, he made sure to stay away from the top of the tree, safe and sound.\n", - "\n", + "Once upon a time there was a little girl named Lucy. She was very excited because she was going to the park. She was going to the park and she saw a big slide. She wanted to go on the slide, but she was too scared to go down.\n", + "Suddenly, Lucy saw a big, scary monster. It was scary and Lucy was scared. She ran to her mom and said, \"Mom, what is that scary monster in the park?\"\n", + "Her mom smiled and said, \"That's a monster, Lucy. It's just a big, scary monster.\"\n", + "The monster said, \"I'm scared of the monster. It's just a monster.\"\n", + "Lucy was scared, but she was brave. She said, \"Don't worry, I'll help you.\"\n", + "The monster was friendly and Lucy was happy to hear the monster. She said, \"Thank you, monster!\"\n", + "The monster smiled and said, \"You're welcome, little one. I'm glad you're safe.\"\n", + "The monster smiled and said, \"You're welcome, Lucy. I'm glad you're safe.\"\n", "\n", - "Step 3600, Loss: 1.4788475036621094, Elapsed Time: 76.91 seconds\n", - "Generated text:\n", - "Once upon a time, there was a little boy named Timmy. Timmy loved to play with his toys and watch cartoons. One day, Timmy's mommy asked him to help his mommy. Timmy was excited to help, so he got a new toy from his grandma's house. \n", - "After a few days, Timmy's mommy gave him a special gift. Timmy was so excited to receive such a gift! It was a special gift that his grandma gave him a big hug. Timmy was so happy to have such a gift and hugged his mommy. \n", - "After that, Timmy went to bed that night with a special gift from his grandma. He loved his present so much that he gave it a special gift to his grandma. Timmy was so happy to have a present that he had found such a beautiful gift. \n", - "The next day, Timmy's grandma gave him a gift and gave him a gift. Timmy was so excited and couldn't wait to tell his grandma. He showed his grandma a gift for him with a present, and they all said thank you to his grandma. Timmy was so happy to receive the gift from her grandma and her gift. \n", - "The present was that Timmy had!!!!\n", "\n", - "Step 3800, Loss: 1.4588173627853394, Elapsed Time: 88.57 seconds\n", + "Step 3600, Loss: 1.4839898347854614, Elapsed Time: 32.21 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Timmy. Timmy loved to play with his toys, but he always wanted to share them with others. One day, Timmy was playing with his toy cars and accidentally broke them. His mom said, \"Don't worry, I will help you fix your toys. Let's make it together.\"\n", - "So, Timmy went to his room and started to play with his toy cars. He had so much fun playing with it, he didn't even want to lose it. His mom was happy to help, so they took a break and put the toy cars away in the right place. Timmy was very happy that he could share his toys with his mom.\n", + "Once upon a time, there was a little girl named Lucy. She was three years old and loved to play outside. One day, Lucy's mommy said she had to go to the store to buy some groceries. Lucy was so excited! She ran to the store and bought some groceries.\n", + "When Lucy got there, she saw a big, red apple. She was so happy! She picked it up and started to eat it. It was so yummy! Lucy was so happy! She ate the apple and felt so good.\n", + "After she finished eating, Lucy's mommy said she had to go home. Lucy was sad because she couldn't play with her toys anymore. She was sad because she couldn't play with her toys anymore.\n", "\n", "\n", - "Step 4000, Loss: 1.4644711017608643, Elapsed Time: 69.17 seconds\n", + "Step 3800, Loss: 1.4641729593276978, Elapsed Time: 31.44 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play in the garden, and even when she saw a beautiful butterfly. She ran over to the butterfly and tried to catch it, but it flew away. \n", - "Suddenly, a little girl appeared. She looked around and saw her and asked her what she was doing. Lily replied, \"I want to play too, please!\" Her mom replied, \"Okay, but only if you don't like the butterfly, it's better.\" \n", - "So, Lily decided to go back to the garden to find the butterfly. She walked over to the garden and saw a big, beautiful butterfly. The butterfly flapped its wings and flew away. Lily felt so happy that she ran back to the garden, happy that she could play with the butterfly again.\n", + "Once upon a time, there was a little girl called Lucy. She was three years old and loved to play outside. One day, she was playing in the garden when she saw a big, shiny rock. She picked it up and showed it to her mom.\n", + "\"Look, Mommy, I found a rock!\" she said.\n", + "Her mom smiled and said, \"That's a great rock, Lucy. It's very pretty.\"\n", + "Lucy was so happy to have a new friend. She showed it to her mom and said, \"Look, Mommy, I found a rock!\"\n", + "Her mom smiled and said, \"That's great, Lucy. Let's keep it safe.\"\n", + "So, they went inside and found a nice spot to keep it safe. They were so happy to have found a rock and they played together all day.\n", "\n", "\n", - "Step 4200, Loss: 1.4640543460845947, Elapsed Time: 72.24 seconds\n", + "Step 4000, Loss: 1.46944260597229, Elapsed Time: 31.43 seconds\n", "Generated text:\n", - "Once upon a time, there was a boy named Timmy. Timmy loved to play with his toy cars, trucks and even the cars. One day, Timmy's toy car got stuck in a big mud. Timmy tried to get the wheels, but he was too heavy. He tried to push the wheel but it was too heavy. Timmy tried and tried, but he couldn't do it. Timmy was sad, but then he remembered how his mom told him. He was brave and strong, but he knew how to use his car to get it. He was happy again, and he was able to get the wheel.\n", - "\n", + "Once upon a time, there was a little girl named Lucy. She was three years old and she loved to play outside. One day, she went to the park with her mom. She saw a big slide and wanted to go on it.\n", + "Lucy went up to the slide and climbed up. She was very excited to go on the slide. She climbed up the ladder and slid down the slide. Whe!\n", + "When she got to the top, she saw a big slide. She was so happy! She climbed up the ladder and slid down the slide. She was so happy!\n", + "When she got to the top, she saw a big slide. She climbed up the ladder and slid down the slide. She was so happy! She laughed and slid down the slide.\n", + "At the end of the day, Lucy was tired but happy. She had a fun day at the park.\n", "\n", - "Step 4400, Loss: 1.4368994235992432, Elapsed Time: 66.23 seconds\n", - "Generated text:\n", - "Once upon a time, there was a little boy named Timmy. Timmy loved to play outside and pick flowers. One day, he saw a little boy who was very curious. He went up to him and asked, \"Can I help you?\" The boy replied, \"Sure, you can help me find your way.\"\n", - "Timmy and the boy searched everywhere but they couldn't find the way up to the boy. They searched everywhere but they couldn't find the answer. Suddenly, the boy said, \"Don't worry, I'll help you find your way.\" Timmy said, \"I know how to solve this problem.\"\n", - "Timmy's face lit up with excitement and said, \"I'll help you solve this problem. I can find your way home.\" His mom said, \"That's right. Let's find your way and see if we can find a way to find my way back.\" Timmy and the boy looked at each other's hand and said, \"Yes, I'll help you.\"\n", - "They found a small tree, and Timmy helped his mom get his help. They worked together to search for the little boy's journey. After a few hours, they finally found the perfect path. Timmy was so happy that he could!!!!\n", "\n", - "Step 4600, Loss: 1.4162362813949585, Elapsed Time: 88.24 seconds\n", + "Step 4200, Loss: 1.4679889678955078, Elapsed Time: 31.67 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Timmy. Timmy loved to play outside and look at the pretty flowers. One day, Timmy saw a beautiful flower. It was so pretty and he wanted to touch it. \n", - "But when he got close to the flower, he accidentally dropped it. It broke! Timmy was so sad that he cried and his mom couldn't fix it. She said they would get another special thing, but Timmy was okay. \n", - "His mom hugged him and said it was okay. She told him that accidents happen and that it's important to be careful when you touch things that can happen.\n", + "Once upon a time there was a little girl named Lucy. She was very happy and loved to play with her friends. One day, she saw a big, red ball in the park. She wanted to play with it, but she was too scared to move.\n", + "Lucy tried to move the ball, but it was too fast. She tried to move it, but it was too fast. She tried to move it, but it was too fast. She tried to move it, but it was too fast.\n", + "Then, a kind lady saw Lucy and asked her if she could help her. She said yes, and Lucy was so happy. She gave Lucy a big hug and a big hug.\n", + "The lady was so happy that she hugged Lucy and thanked her. She said she was a good girl and she was very happy.\n", "\n", "\n", - "Step 4800, Loss: 1.4360578060150146, Elapsed Time: 66.49 seconds\n", + "Step 4400, Loss: 1.4406870603561401, Elapsed Time: 31.44 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play outside in the park. One day, she saw a big tree and wanted to climb it. But her mom told her to be careful because the tree might hurt them.\n", - "Lily tried to climb the tree, but she was too weak. She fell down and started to cry. Her mom hugged her and told her to go back home. They were very happy and went back to the tree.\n", + "Once upon a time, there was a little girl named Lucy. She was very happy and she loved to play with her toys. One day, she was playing with her toys when she saw a big, scary monster. She screamed and ran away.\n", + "Lucy was very scared and ran away. She ran and hid behind a tree. Suddenly, a big dog came running towards her. Lucy was very scared and ran away.\n", + "The big dog saw her and ran away. Lucy was very sad and scared. She ran back to her mom and dad and told them what happened. Her mom hugged her and said, \"It's okay, Lucy. We will always be careful and not hurt you.\"\n", + "The big dog stopped and looked at Lucy. He said, \"I'm sorry, Lucy. I was just scared of you.\"\n", + "Lucy was happy and hugged her mom. She said, \"It's okay, I'm here. I'm glad I was safe.\"\n", "\n", "\n", - "Step 5000, Loss: 1.4377806186676025, Elapsed Time: 59.31 seconds\n", + "Step 4600, Loss: 1.4200118780136108, Elapsed Time: 32.36 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play outside and collect things. One day, she went to the park with her mom. She saw a boy sitting at the bench and he said, \"Hello, doggy!\" Lily felt embarrassed because she didn't like it. She asked the boy, \"What's wrong?\" The boy said, \"My dog's dog bit me!\" Lily said, \"It's too bad. I'm too small.\" The boy felt sad for her.\n", - "Lily said, \"I'm sorry, but I don't know it's not good.\" She went to her mom and asked if she could borrow some of her toys. Her mom said, \"Sure, you can borrow your toy car. You can borrow it, but be careful.\" Lily smiled and went back to playing with her toy car. She had a lot of fun with her toy car and played with it all day.\n", + "Once upon a time, there was a little girl named Jane. She was very happy and loved to play outside. One day, she saw a big, scary monster. It was so scary that she started to scream.\n", + "The monster was so scared that it started to run away. Jane was so scared that she ran away. She ran and ran until she was safe.\n", + "The monster was so scared that it ran away. Jane was so scared that she ran away. She never got to be scared.\n", + "The monster chased Jane and ran away. But she was too fast. She was too scared to run away.\n", + "The monster chased Jane and ran until she was safe. She was safe and happy.\n", "\n", "\n", - "Step 5200, Loss: 1.3954946994781494, Elapsed Time: 76.86 seconds\n", + "Step 4800, Loss: 1.4398597478866577, Elapsed Time: 31.73 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play outside and pick flowers. One day, she saw a beautiful flower in the garden. She picked it up and held it in her hand. She felt so happy that she wanted to show her mom.\n", - "Her mom came outside and saw that Lily was very sad. She told Lily that the flower was still a flower. Lily felt sorry for being so pretty. She promised to be more careful with the flower.\n", - "The next day, Lily went back outside to pick some flowers. She found some pretty flowers and showed them to her mom. Her mom was very happy and said that Lily's flower was not as pretty as the flowers in the garden. From that day on, Lily loved to play and make pretty flowers in the garden.\n", + "Once upon a time there was a little girl named Lucy. She was very happy and loved to play with her friends. One day, she was playing in the park when she saw a big, red ball. She wanted to play with it, so she ran to her mom.\n", + "\"Mom, what is this?\" asked Lucy.\n", + "\"It's a ball, sweetheart,\" her mom replied.\n", + "\"It's a ball, but it's not a toy,\" Lucy said.\n", + "Her mom smiled and said, \"Yes, it's a good idea. Let's go to the park and play with the ball.\"\n", + "So, Lucy and her friends played with the ball all day long. They had so much fun and laughed together.\n", "\n", "\n", - "Step 5400, Loss: 1.401665449142456, Elapsed Time: 70.89 seconds\n", + "Step 5000, Loss: 1.4402837753295898, Elapsed Time: 31.41 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Timmy. Timmy loved to play outside and explore. One day, Timmy saw a big tree with a lot of branches. He thought it looked interesting and decided to climb the tree. \n", - "Timmy's friend, a wise old owl, said, \"Timmy, don't be careless. I don't want to fall. You need to get hurt.\" \n", - "Timmy didn't understand why the owl was so wise, so he explained to his friends, \"It's important to always stay safe.\" \n", - "Timmy felt better and went back to his tree to admire the branches from branch. He knew that even if something is too small, it's always better to be safe and be careful when climbing tree.\n", + "Once upon a time, there was a little girl named Lucy. She was three years old and loved to play in the park. One day, she saw a big, shiny thing in the grass. She wanted to take it home, but it was too heavy for her. She tried to lift it, but it was too heavy. She tried and tried, but it was too heavy. She tried and tried, but it was too heavy. She tried and tried, but it was too heavy. She tried and tried, but it was too heavy. She tried and tried, but it still couldn't. She felt sad and frustrated. She wished she had a friend to help her.\n", "\n", "\n", - "Step 5600, Loss: 1.3960157632827759, Elapsed Time: 71.91 seconds\n", + "Step 5200, Loss: 1.3984873294830322, Elapsed Time: 31.73 seconds\n", + "Generated text:\n", + "Once upon a time, there was a little girl named Lucy. She was three years old and loved to play with her toys. One day, she was playing with her toys when she accidentally broke one of the pieces. She was very sad and didn't know what to do.\n", + "Her mom saw her crying and asked her what was wrong. Lucy told her that she had broken pieces. Her mom said, \"Don't worry, we can fix the pieces.\"\n", + "So Lucy and her mom went to the kitchen and fixed the pieces. When they got to the kitchen, Lucy was so happy and thanked her mom. She said, \"I'm sorry I broke my piece.\"\n", + "Lucy was so sad and cried. She said, \"I'm sorry, Mom. I didn't mean to break my piece.\"\n", + "Her mom hugged her and said, \"It's okay, Lucy. Accidents happen. Let's clean it up again and we can fix it again.\"\n", + "Lucy was so happy and hugged her mom. She hugged her and said, \"I'm sorry, Mom. I didn't mean to break the pieces. I'm sorry I didn't mean to break.\"\n", + "Her mom hugged her and said, \"It's okay, Lucy. I forgive!!!!\n", + "\n", + "Step 5400, Loss: 1.4043035507202148, Elapsed Time: 32.24 seconds\n", "Generated text:\n", - "Once upon a time, a little girl named Lily went on a trip with her mommy. They were having so much fun. When they got to the beach, they saw a crab and wanted to touch it.\n", - "Lily said, \"Mommy, can I touch the crab?\"\n", - "\"No, it's too dangerous. You should stay in the sand. The crab might be safe,\" her mommy replied.\n", - "Lily was sad and started to cry. \"I'm sorry, mommy. I didn't want to be hurt.\"\n", - "Her mommy said, \"It's okay, sweetie. I'm glad you listened to your mommy and not touched the crab. Now you can't touch the crab's safety.\"\n", - "Lily smiled and hugged her mommy. From that day on, they became good friends and they played together every day.\n", + "Once upon a time, there was a little girl named Lucy. She was three years old and loved to play with her toys. One day, she was playing with her toys when she heard a loud noise. She looked around and saw a big, scary monster. The monster was roaring and roaring. Lucy was scared and ran away.\n", + "The monster chased her and Lucy ran away. She ran and hid behind a tree. She never saw the monster again. The monster was gone and Lucy was safe.\n", + "The monster was very strong and brave. He chased Lucy and ran away. Lucy was safe and sound. She never saw the monster again.\n", "\n", "\n", - "Step 5800, Loss: 1.3838456869125366, Elapsed Time: 73.78 seconds\n", + "Step 5600, Loss: 1.3977570533752441, Elapsed Time: 31.21 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Tim. Tim had a big toy car. The car could go very fast. Tim liked to race with his car.\n", - "One day, Tim saw a big box. He wanted to open it. Tim went to his friend, Sue. Sue saw the box and said, \"I can open this box!\" Sue wanted to open the box. They opened the box. Inside, there was a big, soft teddy bear. The bear was happy.\n", - "Tim and Sue played with the bear all day. They were not very good at all. They played with the toys and had lots of fun. They learned that being kind was better and to help others. And from that day on, Tim and Sue were the best of friends.\n", + "Once upon a time, there was a little girl named Lucy. She was very excited because she was going to the park. She was going to the park and she was going to the park.\n", + "When she arrived at the park, she saw a big slide. She wanted to slide down the slide, but she was scared. She started to cry.\n", + "Suddenly, a voice came from the park. It was a little girl who was crying. She had lost her toy. She looked around and saw a man. He was crying and crying.\n", + "The little girl asked the man if he had seen her toy. The man said, \"I'm sorry, but I can't find my toy.\"\n", + "The little girl was so happy and thanked the man. She said, \"I'm sorry I lost my toy. I can't find it.\"\n", + "The man smiled and said, \"It's okay. I'm here. I'll help you find your toy.\"\n", + "The little girl was so happy and thanked the man. She went back home and told her mom about the park. Her mom was very happy and said, \"I'm glad you're here. I'm glad you're here.\"\n", "\n", "\n", - "Step 6000, Loss: 1.4077720642089844, Elapsed Time: 70.17 seconds\n", + "Step 5800, Loss: 1.38675856590271, Elapsed Time: 32.10 seconds\n", "Generated text:\n", - "Once upon a time, a little girl named Sue and her dog, Spot. They lived in a big house with their mom and dad. One day, they found a shiny rock that they had to take home. They took it home and put it in their room.\n", - "Sue's mom saw the shiny rock and said, \"Oh, my toy! Your toy is very special! You should take it home.\" Sue was happy to have her toy back and said, \"Thank you, Spot!\"\n", - "But then, a big wind came and blew the rock back. Sue and Spot got scared and started to cry. Their mom came and saw what happened. She told them not to worry and they both went to the toy store to get the shiny rock.\n", - "The moral of the story is that it's important to take care of our things, but we can also take care of them. If you take care of them, we might find them and help them find them.\n", + "Once upon a time, there was a little girl named Lucy. She was three years old and loved to play outside. One day, Lucy went to the park with her mom. She saw a big, red ball and wanted to play with it.\n", + "Lucy ran to her mom and said, \"Mommy, can I play with the ball?\"\n", + "Her mom said, \"No, Lucy. It's not safe. It's not safe to play with.\"\n", + "Lucy was sad, but she understood. She said, \"Okay, Mommy. I will play with the ball.\"\n", + "Her mom smiled and said, \"Okay, Lucy. Let's play with the ball.\"\n", + "So Lucy and her mom played with the ball and had lots of fun. They had a lot of fun and Lucy was happy.\n", "\n", "\n", - "Step 6200, Loss: 1.3794877529144287, Elapsed Time: 77.59 seconds\n", + "Step 6000, Loss: 1.4106464385986328, Elapsed Time: 31.60 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play outside and pick flowers. One day, she went to the park with her mommy. She saw a boy crying because he lost his toy car.\n", - "Lily said, \"I lost my toy car, but I can't find it.\"\n", - "Lily asked the boy, \"Have you seen my toy car?\" The boy said, \"No, I haven't seen it.\"\n", - "Lily felt sad and went to play on the swings. She went to the boy and said, \"Thank you, you're my friend.\" The boy said, \"You are very kind. Can you help me find your toy car?\"\n", - "Lily smiled and said, \"I will help you find your toy car. Maybe you can find your toy car.\"\n", + "Once upon a time, there was a little girl named Lucy. She loved to play with her toys and her friends. One day, Lucy's mom asked her to clean up her toys. Lucy didn't want to clean up, so she said, \"No, Mommy! I don't want to clean up.\"\n", + "Her mom said, \"But Lucy, you need to clean up your toys. You can do it if you don't clean up.\"\n", + "Lucy didn't want to clean up, so she said, \"But Mommy, I want to clean up my toys. I want to clean up my toys.\"\n", + "Her mom said, \"Okay, but you have to clean up your toys first. You can do it again tomorrow.\"\n", + "Lucy nodded and said, \"Okay, Mommy. I will clean up.\"\n", + "So, Lucy and her mom cleaned up and cleaned up all the toys. They were happy and clean.\n", "\n", "\n", - "Step 6400, Loss: 1.3905563354492188, Elapsed Time: 73.45 seconds\n", + "Step 6200, Loss: 1.3815696239471436, Elapsed Time: 31.65 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play outside in the sun. One day, she saw a big, round thing on the ground. It was a pretty, shiny rock.\n", - "Lily picked up the rock and put it on her foot. She was so happy and showed her mom the rock. Her mom was proud of her and gave her a big hug. Lily felt happy too.\n", - "Later that day, Lily and her mom went to the park. Lily saw a big slide. She wanted to slide down it too. She ran up the slide, and slid down fast. The wind blew and Lily fell. She was hurt and sad.\n", - "Lily learned to be careful when she slide. She went to the swings, and when she was playing, she felt much better. Her mom gave her a hug and told her to keep the smooth rock. Lily learned that she should always keep the rock safe.\n", + "Once upon a time, there was a little girl named Lily. She loved to play outside in the park. One day, she saw a big, scary dog. The dog was very fast and ran away.\n", + "Lily was scared and ran after the dog. She ran as fast as she could. The dog was fast and ran away.\n", + "Lily was safe and happy. She learned that it's important to be careful when she played in the park. She also learned that it's important to be careful when playing in the park.\n", "\n", "\n", - "Step 6600, Loss: 1.39736008644104, Elapsed Time: 76.19 seconds\n", + "Step 6400, Loss: 1.3922070264816284, Elapsed Time: 31.11 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play with her toys and her favorite toy was a teddy bear. One day, Lily was playing with her teddy bear when she saw a shiny object on the ground. She picked it up and looked at it closely. \n", - "\"Wow, look at that object!\" said Lily.\n", - "\"I want it!\" said her mom. \"It's a special object. It's shiny and it's very special to me.\"\n", - "Lily thought for a moment and then decided to take the object from her mom to her friend. \"Look, I found a pretty bracelet!\" said Lily. \n", - "Her friend said, \"That's great, Lily! Let's play with it.\" And so, they played with the bracelet all day long. They were having so much fun!\n", + "Once upon a time, there was a little girl named Lucy. She was three years old and loved to play with her toys. One day, she was playing with her toys when she heard a loud noise. She looked up and saw a big, scary monster!\n", + "The monster was very scary and it started to shake. Lucy was scared and didn't know what to do. She wanted to run away, but the monster was too fast.\n", + "Suddenly, the monster started to shake and growl. Lucy was scared and ran away. She never saw the monster again.\n", + "The monster was gone forever. Lucy was very sad and wished she had never gone to play with her toys. She never forgot the monster and the monster was never seen again.\n", "\n", "\n", - "Step 6800, Loss: 1.4118915796279907, Elapsed Time: 72.23 seconds\n", + "Step 6600, Loss: 1.3984136581420898, Elapsed Time: 31.23 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to draw with her toys and make pictures. One day, she was playing in her backyard when she saw a big bird sitting on a branch. The bird was so pretty! Lily wanted to be friends with the bird, so she tried to sing with it. But the bird didn't fly away. Lily was sad and didn't know what to do.\n", - "Then, Lily saw a bird that it was very graceful. She asked the bird what was wrong. The bird said that it had wings to fly away. Lily thought that it was a fun idea and flew away. She was happy again and went back to her house to play. From that day on, Lily knew that if she wanted to be friends with her friend, she could help the bird feel better.\n", + "Once upon a time, there was a little girl named Lucy. She was three years old and loved to play with her toys. One day, she was playing with her toys when she heard a loud noise. She looked up and saw a big, scary monster. She was scared and didn't know what to do.\n", + "Suddenly, a big, scary monster appeared. It was the monster's owner, who was watching her. She was scared and ran away. Lucy was scared and ran home.\n", + "The monster was very angry and started to chase her. Lucy was scared and ran as fast as she could. She ran as fast as she could, but the monster was too fast and caught her.\n", + "The monster was caught and took her away. Lucy was very sad and cried. She never saw her toys again.\n", "\n", "\n", - "Step 7000, Loss: 1.4061782360076904, Elapsed Time: 72.51 seconds\n", + "Step 6800, Loss: 1.4159687757492065, Elapsed Time: 31.67 seconds\n", + "Generated text:\n", + "Once upon a time, there was a little girl named Lucy. She loved to play with her toys, but one day she found a big box in the attic. She was so excited to open it and see what was inside.\n", + "Inside the box, she found a shiny new toy. It was a toy car. She wanted to play with it, but it was too expensive. She asked her mom if she could have it, but her mom said no.\n", + "Lucy was sad and didn't know what to do. She wanted to play with the car, but her mom said no. She said it was too expensive and she should not have it.\n", + "Lucy was sad and didn't know what to do. She wanted to play with the car, but her mom said no. Lucy was sad and didn't understand why her mom was so sad.\n", + "Then, her mom came to her room and saw the car. She was so happy and hugged Lucy. She hugged her mom and said, \"I'm sorry, Mom. I didn't mean to spoil you.\"\n", + "Lucy smiled and hugged her mom. She was so happy and grateful. She learned that sometimes things can be expensive, but it's important to be kind and share with others.\n", + "!!!!\n", + "\n", + "Step 7000, Loss: 1.406973123550415, Elapsed Time: 32.53 seconds\n", "Generated text:\n", - "Once upon a time, a little girl named Sue lived in a big house. Sue had a toy car that she loved to play with. One day, Sue's mom told her not to clean up. Sue listened to her mom and started to clean her room.\n", - "After cleaning, Sue felt tired and needed to take a nap. She put on her clean, but her mom was not looking well. Sue went to her room and sat down to rest. Her mom said, \"Sue, I'm going to rest and drink a big drink.\" Sue smiled and felt better.\n", - "Sue went to bed with her mom to rest her room. In the end, she went to sleep with her mom. She felt happy that she was able to help her mom. From that day on, Sue knew that her mom loved her because she loved her little sister so much.\n", + "Once upon a time, there was a little girl named Lucy. She was three years old and loved to play outside. One day, Lucy was playing in the park when she saw a big, scary dog. She was scared and ran away.\n", + "Lucy was very scared. She didn't know what to do. She looked around and saw a big, scary dog. The dog was friendly and wagged its tail. Lucy was so scared that she ran away.\n", + "The dog was very friendly and wagged its tail. Lucy was so happy that she had a friend. She hugged the dog and said, \"I'm here to play with you!\"\n", + "The dog was so happy that Lucy had a friend. They played together all day long. They laughed and laughed until it was time to go home.\n", "\n", "\n", - "Step 7200, Loss: 1.381988286972046, Elapsed Time: 73.38 seconds\n", + "Step 7200, Loss: 1.3830863237380981, Elapsed Time: 31.75 seconds\n", "Generated text:\n", - "Once upon a time, a little girl named Lily went to the park with her mommy. She saw a boy who looked sad. She went to her mommy and asked her if she could have a turn to her.\n", - "\"Sure, but you have to ask your mommy,\" her mommy said. \"It's okay, but it's important to ask nicely.\"\n", - "Lily nodded and gave the boy a hug. \"Can we play on the swings?\" she asked.\n", - "The boy didn't like it and said, \"Sure, but be careful.\"\n", - "Lily felt happy and went to play on the swings. She went back to the boy and asked if he wanted to swing too. The boy said, \"I want to swing like you, but it's not safe.\"\n", - "Lily went to the swing and pushed the boy away. She was sad that she lost her favorite toy. The boy was upset and didn't want to swing anymore. They both sat on the swing and watched the boy play on the swings. Lily felt better and said, \"Thank you for letting me swing, Timmy. You're my friend.\"\n", + "Once upon a time, there was a little girl named Lucy. She was three years old and loved to play outside.\n", + "One day, Lucy was playing in the park when she saw a big, scary dog. She was scared and ran away.\n", + "Lucy's mom saw her and said, \"Don't worry, Lucy. The dog is just a little bit. He is just playing.\"\n", + "Lucy was so happy that she started to laugh and play. She ran around the park, laughing and having fun.\n", + "Suddenly, Lucy heard a loud noise. She looked up and saw a big, scary dog. Lucy was scared and ran back home.\n", + "The next day, Lucy was playing in the park again. She was so scared that she ran back home.\n", "\n", "\n", - "Step 7400, Loss: 1.3883332014083862, Elapsed Time: 83.07 seconds\n", + "Step 7400, Loss: 1.390000581741333, Elapsed Time: 31.51 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play outside in the garden and feel the warm sunshine on her skin. One day, while she was playing, she noticed that the flowers had bloomed in a beautiful butterfly. She wanted to see the butterfly so she asked her mommy if she could go to the butterfly's home. Her mommy said yes and Lily ran to her mommy's room. \n", - "When she got there, she saw that the butterfly was sad and started to cry. Her mommy told her that she couldn't go back to the garden to get her dress and her mommy was not happy. Lily felt bad and didn't want to go back to the garden to get her dress. \n", - "The next day, Lily saw the butterfly again and ran back to her mommy's house. She was happy to see her mommy again and said she was a great helper. Her mommy was so happy that she could see the butterfly again and that was a beautiful butterfly who was a good girl.\n", + "Once upon a time, there was a little girl named Lucy. She was very happy and loved to play outside. One day, she went to the park with her mommy and daddy. She saw a big tree and wanted to climb it. She asked her mommy if she could go. Her mommy said yes and Lucy was so excited. She ran up the tree and started to climb. She was so high that she could see the whole park. She was so happy to be outside and she wanted to go home. She ran and ran until she reached the top of the tree. She looked around and saw a big, beautiful butterfly. She was so happy to see it and she wanted to touch it. She ran back to her mommy and daddy and told them about the butterfly. They said they were so lucky to see it and they all hugged. Lucy was so happy to be home with her new friend.\n", "\n", "\n", - "Step 7600, Loss: 1.3650509119033813, Elapsed Time: 80.14 seconds\n", + "Step 7600, Loss: 1.3659923076629639, Elapsed Time: 31.61 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play with her toys, but one day she got bored. She went to her friend, Tim, and asked if he could borrow his toy.\n", - "Tim said, \"No, Lily. This is my toy car. You can play with it, but I want to borrow your toy car.\"\n", - "Lily felt sad because she didn't want to play with Tim. She said, \"Please, Tim. We can borrow your toy car. I can borrow it for my birthday.\"\n", - "Tim took the toy car and said, \"No, Lily. I can borrow it for my birthday.\" He played with the toy car all day. He was happy with his new toy car.\n", + "Once upon a time, there was a little girl named Amy. She was three years old and loved to play. One day, she was playing in the park when she saw a big, shiny ball. She wanted to play with it, so she ran to get it.\n", + "When she got close, she saw a big, shiny ball. She was so excited and wanted to play with it. She ran to the ball and picked it up. She threw the ball and it went very fast.\n", + "When she got back to the park, she saw a big, shiny ball. She was so happy and ran to play with it. She kicked the ball and it flew high in the sky. She was so happy and she had so much fun with the big, shiny ball.\n", "\n", "\n", - "Step 7800, Loss: 1.3602105379104614, Elapsed Time: 69.70 seconds\n", + "Step 7800, Loss: 1.361992359161377, Elapsed Time: 31.49 seconds\n", "Generated text:\n", - "Once upon a time, a little girl named Lily went to a park with her mommy. The park was a pretty green place, with a big pond with lots of ducks. Lily loved the sound of the water, so she decided to play in the water.\n", - "Lily saw a butterfly flying in the sky and tried to catch it. But the butterfly was too fast, and she couldn't catch it. Suddenly, the butterfly landed on a flower and Lily started to panic.\n", - "Her mommy saw the butterfly and asked, \"What happened, Lily?\"\n", - "\"The butterfly fell and hurt my hand,\" said Lily.\n", - "Her mommy came over and saw the butterfly and asked, \"Are you okay, Lily?\"\n", - "\"It hurts, sweetie,\" said her mommy.\n", - "Lily's mommy took a big spoon and the butterfly was very pretty and had lots of colors on it. Lily felt much better and was able to catch the butterfly. She was happy to have her friend a little friend, and they played together all day long.\n", + "Once upon a time, there was a little girl named Lucy. She was very excited because today was a special day. She had a big box of makeup that she wanted to try and play with.\n", + "So, she put on her makeup and went outside to play. She saw a big, scary monster! The monster was very scary and it was so scary.\n", + "Lucy was scared and started to cry. She wanted to get her makeup, but she was too scared to go inside. She tried to run away, but the monster was too fast.\n", + "Then, the monster came closer and closer. It bit Lucy's finger and she was so scared. She screamed and cried until the monster was gone.\n", + "The monster was so scared that it ran away. Lucy was so sad and scared. She wished she had been more careful.\n", + "The next day, the monster came back to Lucy and she was so happy. She had a new friend and she was so happy to be safe.\n", "\n", "\n", - "Step 8000, Loss: 1.3417373895645142, Elapsed Time: 79.57 seconds\n", + "Step 8000, Loss: 1.3433030843734741, Elapsed Time: 31.83 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Tim. Tim had a toy named Sam. Sam liked to play with the toy car. One day, Tim's mom said they were going to a new park. Tim wanted to play on the swings, but his mom said no.\n", - "Tim started playing with the car. He saw a big dog in the park. The dog barked loudly. Tim felt scared. He wanted to run away. He went to his mom and dad and said they were safe. Tim felt safe.\n", - "Later, Tim and Sam went to the park. They saw a big slide. They both wanted to go on the slide. They ran to the slide and slid down the slide. Tim's mom and dad helped them get down. They all laughed and had a great time together.\n", + "Once upon a time, there was a little girl named Lucy. She was very excited because she was going to the park. She ran to the park and saw a big tree. She wanted to climb it.\n", + "She asked her mom if she could climb the tree. Her mom said yes, so she climbed the tree. She climbed higher and higher until she reached the top.\n", + "When she got to the top, she saw a big tree. She was so happy! She climbed up the tree and climbed up. She felt so good on the top.\n", + "When she got to the top, she saw a big, beautiful view. She was so happy! She ran to the top and looked down. She felt so lucky to be able to climb the tree.\n", + "Lucy was so glad she got to climb the tree. She was so glad she got to climb the tree. She was so glad she got to climb the tree.\n", "\n", "\n", - "Step 8200, Loss: 1.3874244689941406, Elapsed Time: 71.68 seconds\n", + "Step 8200, Loss: 1.3881804943084717, Elapsed Time: 31.76 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Timmy. Timmy loved to play with his toy car, but his favorite thing to do was to go to the store. Timmy didn't like the store, so he went to the store.\n", - "When they got to the store, Timmy saw a toy car and wanted it. He asked his mom for a toy car and her mom said, \"No, it's too expensive.\" Timmy didn't want to buy a toy car. He thought the toy car wouldn't have a toy car, but his mom said it was okay.\n", - "Later that day, Timmy went to the park with his mom. Timmy saw a boy who looked sad because he lost his toy car. Timmy felt sorry for the boy and said, \"I can help you find my toy car.\" But he couldn't find his toy car, and he didn't know what to do.\n", - "Timmy asked his mom, \"Why are you looking for my toy car, and the boy can have it back?\" His mom said, \"Don't worry, we can fix it. Let's find your toy car.\" They found the toy car and it was in the store. Timmy was happy that he found a!!!!\n", - "\n", + "Once upon a time, there was a little girl named Lucy. She was very happy and loved to play with her friends. One day, Lucy's mom asked her to help her. Lucy was very excited and said yes.\n", + "Lucy's mom said, \"Let's go to the store and buy some food.\" Lucy was very happy and said, \"Yay! I want to buy some food!\"\n", + "So, Lucy and her mom went to the store to buy food. When they got to the store, Lucy saw a big, yummy food. She asked her mom if she could buy it. Her mom said, \"No, Lucy. We can't buy it.\"\n", + "Lucy was sad and said, \"But I want to buy something else. I want to buy something else.\"\n", + "Her mom said, \"Okay, but you have to wait until you get to buy something else. It's a nice, but you have to wait until it's time to get something else.\"\n", + "Lucy was so excited and she waited patiently. When it was time to go, she was so happy and thanked her mom for the yummy food.\n", "Final generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play outside and pick flowers. One day, Lily saw a butterfly and tried to catch it, but it flew away. Lily felt sad because she didn't know what was happening.\n", - "Suddenly, her friend, a little bird named Timmy, came to visit. \"Don't worry, Timmy,\" said Lily. \"I can catch it!\"\n", - "\"I'm sorry, Lily,\" said Timmy. \"I just need to help my mom.\"\n", - "\"Thank you, Timmy,\" said Lily. \"You are very kind to me.\"\n", - "From that day on, Lily learned to always help others when they needed it. And every time she saw the butterfly, she felt happy again.\n", - "\n" + "Once upon a time, there was a little girl named Lucy. She was very excited to go to the park. She put on her shoes and ran to the park.\n", + "When she got to the park, she saw a big slide. She wanted to go on it. She ran to the slide and started to slide down. She was so fast!\n", + "Suddenly, she heard a loud noise. It was a big, scary dog. The dog was barking loudly. Lucy was scared. She ran back to the park and started to run.\n", + "The dog was so fast that it ran away. Lucy was safe. She was so happy she had gone on the slide.\n" ] } ], @@ -986,11 +988,10 @@ "\n", "start_prompt = \"Once upon a time\"\n", "start_tokens = tokenizer.encode(start_prompt)[:maxlen]\n", + "print(f\"Initial generated text:\")\n", "generated_text = model.generate_text(\n", " maxlen, start_tokens\n", ")\n", - "print(f\"Initial generated text:\\n{generated_text}\\n\")\n", - "\n", "\n", "metrics_history = {\n", " 'train_loss': [],\n", @@ -1014,20 +1015,21 @@ " metrics.reset()\n", "\n", " elapsed_time = time.time() - start_time\n", - " print(f\"Step {step + 1}, Loss: {metrics_history['train_loss'][-1]}, Elapsed Time: {elapsed_time:.2f} seconds\")\n", + " print(f\"\\n\\nStep {step + 1}, Loss: {metrics_history['train_loss'][-1]}, Elapsed Time: {elapsed_time:.2f} seconds\")\n", " start_time = time.time()\n", "\n", + " print(f\"Generated text:\")\n", " generated_text = model.generate_text(\n", " maxlen, start_tokens\n", " )\n", - " print(f\"Generated text:\\n{generated_text}\\n\")\n", + "\n", " step += 1\n", "\n", "# Final text generation\n", + "print(f\"Final generated text:\")\n", "generated_text = model.generate_text(\n", " maxlen, start_tokens\n", - ")\n", - "print(f\"Final generated text:\\n{generated_text}\")" + ")" ] }, { @@ -1048,12 +1050,12 @@ "height": 472 }, "id": "B6Eg1Cz2y_iP", - "outputId": "7cafe711-1ae4-4eb9-fd37-e1bde54cbfc5" + "outputId": "fb96b456-23a8-448f-ebc6-23d807a626d2" }, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABIjElEQVR4nO3deXxU9b3/8fdMlhmyTBYgC5CETQmLYRMhUEUFQaW9IGiV6g9woVXhXqy219JeN6wX1Kut1opYW7EqUqWC1WoF0WAVkMWAbCIoJAGSIEsyWSfJzPn9ETIYgZCEmTmZyev5cB5kzpwz8zk54rz9nu9iMQzDEAAAQIiwml0AAACALxFuAABASCHcAACAkEK4AQAAIYVwAwAAQgrhBgAAhBTCDQAACCmEGwAAEFIINwAAIKQQbgD43YwZM9S9e/dWHfvggw/KYrH4tiAAIY1wA7RjFoulWY+cnByzSzXFjBkzFBMTY3YZAFrIwtpSQPv1yiuvNHr+17/+VatWrdLLL7/caPsVV1yh5OTkVn9ObW2tPB6PbDZbi4+tq6tTXV2d7HZ7qz+/tWbMmKFly5apvLw84J8NoPXCzS4AgHluuummRs/Xr1+vVatWnbL9+yorKxUVFdXsz4mIiGhVfZIUHh6u8HD+UwWg+bgtBaBJl156qQYMGKDNmzfrkksuUVRUlH79619Lkt566y1NmDBBXbp0kc1mU69evfTwww/L7XY3eo/v97nZv3+/LBaL/u///k/PP/+8evXqJZvNpmHDhmnjxo2Njj1dnxuLxaLZs2drxYoVGjBggGw2m/r3769//etfp9Sfk5OjCy+8UHa7Xb169dKiRYt83o/njTfe0NChQ9WhQwd16tRJN910kw4ePNhon6KiIt18883q1q2bbDabUlNTNXHiRO3fv9+7z6ZNmzR+/Hh16tRJHTp0UI8ePXTLLbf4rE6gveB/hwCc1dGjR3XVVVfphhtu0E033eS9RbV48WLFxMTo7rvvVkxMjD788EPdf//9cjqdevzxx8/6vkuWLFFZWZl+9rOfyWKx6LHHHtPkyZP1zTffnLW155NPPtGbb76pO++8U7GxsXr66ac1ZcoU5efnq2PHjpKk3NxcXXnllUpNTdVDDz0kt9utefPmqXPnzuf+Szlh8eLFuvnmmzVs2DDNnz9fxcXFeuqpp/Tpp58qNzdX8fHxkqQpU6Zox44d+s///E91795dhw8f1qpVq5Sfn+99Pm7cOHXu3Fm/+tWvFB8fr/379+vNN9/0Wa1Au2EAwAmzZs0yvv+fhdGjRxuSjOeee+6U/SsrK0/Z9rOf/cyIiooyqqurvdumT59uZGRkeJ/v27fPkGR07NjROHbsmHf7W2+9ZUgy3n77be+2Bx544JSaJBmRkZHG3r17vdu2bt1qSDL+8Ic/eLf96Ec/MqKiooyDBw96t+3Zs8cIDw8/5T1PZ/r06UZ0dPQZX6+pqTGSkpKMAQMGGFVVVd7t77zzjiHJuP/++w3DMIzjx48bkozHH3/8jO+1fPlyQ5KxcePGs9YFoGnclgJwVjabTTfffPMp2zt06OD9uaysTEeOHNHFF1+syspKffnll2d93+uvv14JCQne5xdffLEk6ZtvvjnrsWPHjlWvXr28z7OysuRwOLzHut1uffDBB5o0aZK6dOni3a9379666qqrzvr+zbFp0yYdPnxYd955Z6MOzxMmTFBmZqb++c9/Sqr/PUVGRionJ0fHjx8/7Xs1tPC88847qq2t9Ul9QHtFuAFwVl27dlVkZOQp23fs2KFrrrlGcXFxcjgc6ty5s7czcmlp6VnfNz09vdHzhqBzpgDQ1LENxzcce/jwYVVVVal3796n7He6ba2Rl5cnSerTp88pr2VmZnpft9lsevTRR/Xee+8pOTlZl1xyiR577DEVFRV59x89erSmTJmihx56SJ06ddLEiRP14osvyuVy+aRWoD0h3AA4q++20DQoKSnR6NGjtXXrVs2bN09vv/22Vq1apUcffVSS5PF4zvq+YWFhp91uNGOGinM51gx33XWXvvrqK82fP192u1333Xef+vbtq9zcXEn1naSXLVumdevWafbs2Tp48KBuueUWDR06lKHoQAsRbgC0Sk5Ojo4eParFixdrzpw5+uEPf6ixY8c2us1kpqSkJNntdu3du/eU1063rTUyMjIkSbt37z7ltd27d3tfb9CrVy/dc889WrlypbZv366amho98cQTjfYZMWKEHnnkEW3atEmvvvqqduzYoaVLl/qkXqC9INwAaJWGlpPvtpTU1NTo2WefNaukRsLCwjR27FitWLFChw4d8m7fu3ev3nvvPZ98xoUXXqikpCQ999xzjW4fvffee9q1a5cmTJggqX5eoOrq6kbH9urVS7Gxsd7jjh8/fkqr06BBgySJW1NACzEUHECrjBw5UgkJCZo+fbr+67/+SxaLRS+//HKbui304IMPauXKlRo1apTuuOMOud1uPfPMMxowYIC2bNnSrPeora3Vb3/721O2JyYm6s4779Sjjz6qm2++WaNHj9bUqVO9Q8G7d++un//855Kkr776SmPGjNGPf/xj9evXT+Hh4Vq+fLmKi4t1ww03SJJeeuklPfvss7rmmmvUq1cvlZWV6U9/+pMcDoeuvvpqn/1OgPaAcAOgVTp27Kh33nlH99xzj/7nf/5HCQkJuummmzRmzBiNHz/e7PIkSUOHDtV7772nX/ziF7rvvvuUlpamefPmadeuXc0azSXVt0bdd999p2zv1auX7rzzTs2YMUNRUVFasGCB7r33XkVHR+uaa67Ro48+6h0BlZaWpqlTp2r16tV6+eWXFR4erszMTL3++uuaMmWKpPoOxRs2bNDSpUtVXFysuLg4XXTRRXr11VfVo0cPn/1OgPaAtaUAtDuTJk3Sjh07tGfPHrNLAeAH9LkBENKqqqoaPd+zZ4/effddXXrppeYUBMDvaLkBENJSU1M1Y8YM9ezZU3l5eVq4cKFcLpdyc3N13nnnmV0eAD+gzw2AkHbllVfqtddeU1FRkWw2m7Kzs/W///u/BBsghNFyAwAAQgp9bgAAQEgh3AAAgJDS7vrceDweHTp0SLGxsbJYLGaXAwAAmsEwDJWVlalLly6yWptum2l34ebQoUNKS0szuwwAANAKBQUF6tatW5P7tLtwExsbK6n+l+NwOEyuBgAANIfT6VRaWpr3e7wp7S7cNNyKcjgchBsAAIJMc7qU0KEYAACEFMINAAAIKYQbAAAQUgg3AAAgpBBuAABASCHcAACAkEK4AQAAIYVwAwAAQgrhBgAAhBTCDQAACCmEGwAAEFIINwAAIKQQbnyk1u1RsbNaBccqzS4FAIB2jXDjI5v2H9fw/12tGS9uMLsUAADatTYTbhYsWCCLxaK77rrrjPssXrxYFoul0cNutweuyCY4OoRLkpzVdSZXAgBA+xZudgGStHHjRi1atEhZWVln3dfhcGj37t3e5xaLxZ+lNVtchwhJkrOq1uRKAABo30xvuSkvL9eNN96oP/3pT0pISDjr/haLRSkpKd5HcnJyAKo8O8eJcOOq86i61m1yNQAAtF+mh5tZs2ZpwoQJGjt2bLP2Ly8vV0ZGhtLS0jRx4kTt2LGjyf1dLpecTmejhz/ERIaroRGpjFtTAACYxtRws3TpUn3++eeaP39+s/bv06eP/vKXv+itt97SK6+8Io/Ho5EjR+rAgQNnPGb+/PmKi4vzPtLS0nxVfiNWq0WxtoZ+N9yaAgDALKaFm4KCAs2ZM0evvvpqszsFZ2dna9q0aRo0aJBGjx6tN998U507d9aiRYvOeMzcuXNVWlrqfRQUFPjqFE7hoN8NAACmM61D8ebNm3X48GENGTLEu83tduvjjz/WM888I5fLpbCwsCbfIyIiQoMHD9bevXvPuI/NZpPNZvNZ3U1x2CMkVTFiCgAAE5kWbsaMGaNt27Y12nbzzTcrMzNT995771mDjVQfhrZt26arr77aX2W2iHc4OC03AACYxrRwExsbqwEDBjTaFh0drY4dO3q3T5s2TV27dvX2yZk3b55GjBih3r17q6SkRI8//rjy8vJ02223Bbz+06lvuaHPDQAAZmoT89ycSX5+vqzWk92Cjh8/rpkzZ6qoqEgJCQkaOnSo1q5dq379+plY5UkNfW5KabkBAMA0bSrc5OTkNPn8d7/7nX73u98FrqAW8rbcVNHnBgAAs5g+z00o8c5SzG0pAABMQ7jxIToUAwBgPsKND53sUMxtKQAAzEK48SEm8QMAwHyEGx9y2Fl+AQAAsxFufOhkyw23pQAAMAvhxoccjJYCAMB0hBsfargtVVPnUXWt2+RqAABonwg3PhQdGS6rpf5nOhUDAGAOwo0PWa0WxbK+FAAApiLc+Ficd30pOhUDAGAGwo2PeWcppuUGAABTEG587OTimYQbAADMQLjxMZZgAADAXIQbH2PxTAAAzEW48TEHo6UAADAV4cbHWIIBAABzEW58jMUzAQAwF+HGx0623BBuAAAwA+HGxxgKDgCAuQg3PhYXxVBwAADMRLjxMVpuAAAwF+HGx767/IJhGCZXAwBA+0O48bGGlptat6HqWo/J1QAA0P4QbnwsKjJMYVaLJIaDAwBgBsKNj1kslpNz3dDvBgCAgCPc+IF3rhtabgAACDjCjR+cHDHFcHAAAAKNcOMHDSOmSrktBQBAwBFu/ICVwQEAMA/hxg+YyA8AAPMQbvyAJRgAADAP4cYPGAoOAIB5CDd+wFBwAADMQ7jxA4aCAwBgHsKNH3x38UwAABBYhBs/YLQUAADmIdz4wck+N9yWAgAg0Ag3ftDQclNaVSvDMEyuBgCA9oVw4wcNfW7cHkOVNW6TqwEAoH0h3PhBh4gwhVstkuhUDABAoLWZcLNgwQJZLBbdddddTe73xhtvKDMzU3a7XRdccIHefffdwBTYAhaLRXEdGA4OAIAZ2kS42bhxoxYtWqSsrKwm91u7dq2mTp2qW2+9Vbm5uZo0aZImTZqk7du3B6jS5mMiPwAAzGF6uCkvL9eNN96oP/3pT0pISGhy36eeekpXXnmlfvnLX6pv3756+OGHNWTIED3zzDMBqrb5WIIBAABzmB5uZs2apQkTJmjs2LFn3XfdunWn7Dd+/HitW7fOX+W1Gi03AACYI9zMD1+6dKk+//xzbdy4sVn7FxUVKTk5udG25ORkFRUVnfEYl8sll8vlfe50OltXbAuxBAMAAOYwreWmoKBAc+bM0auvviq73e63z5k/f77i4uK8j7S0NL991nd5l2DgthQAAAFlWrjZvHmzDh8+rCFDhig8PFzh4eFas2aNnn76aYWHh8vtPnV+mJSUFBUXFzfaVlxcrJSUlDN+zty5c1VaWup9FBQU+PxcTsfbcsNtKQAAAsq021JjxozRtm3bGm27+eablZmZqXvvvVdhYWGnHJOdna3Vq1c3Gi6+atUqZWdnn/FzbDabbDabz+puroY+N6W03AAAEFCmhZvY2FgNGDCg0bbo6Gh17NjRu33atGnq2rWr5s+fL0maM2eORo8erSeeeEITJkzQ0qVLtWnTJj3//PMBr/9sTo6Wos8NAACBZPpoqabk5+ersLDQ+3zkyJFasmSJnn/+eQ0cOFDLli3TihUrTglJbQGjpQAAMIepo6W+Lycnp8nnknTdddfpuuuuC0xB54BwAwCAOdp0y00wYyg4AADmINz4SVzDUHBabgAACCjCjZ+cbLmplWEYJlcDAED7Qbjxk4Y+Nx5Dqqg5dc4eAADgH4QbP7GFWxUZVv/rZZZiAAACh3DjJxaL5eQSDPS7AQAgYAg3ftTQ76a0knADAECgEG78KNY71w3DwQEACBTCjR+dXIKBlhsAAAKFcONHccxSDABAwBFu/Mi7BAOzFAMAEDCEGz/yTuRHyw0AAAFDuPEj71Bw+twAABAwhBs/ouUGAIDAI9z4EX1uAAAIPMKNH3mHgtNyAwBAwBBu/Kih5aaUPjcAAAQM4caPvH1uCDcAAAQM4caPGkZLlbnq5PEYJlcDAED7QLjxo4aWG8OQymvoVAwAQCAQbvzIHhEmW3j9r5hbUwAABAbhxs8YDg4AQGARbvyM4eAAAAQW4cbPTrbcEG4AAAgEwo2fnVyCgdtSAAAEAuHGz2i5AQAgsAg3ftbQ54ZZigEACAzCjZ95W27oUAwAQEAQbvzs5BIM9LkBACAQCDd+FkfLDQAAAUW48bOG9aXoUAwAQGAQbvyMoeAAAAQW4cbPGAoOAEBgEW78jOUXAAAILMKNnzW03JS76uTxGCZXAwBA6CPc+FnsiZYbw5DKXPS7AQDA3wg3fmYLD5M9ov7XTL8bAAD8j3ATAA0jpliCAQAA/yPcBABLMAAAEDiEmwDwzlLMEgwAAPgd4SYAGA4OAEDgEG4CgIn8AAAIHFPDzcKFC5WVlSWHwyGHw6Hs7Gy99957Z9x/8eLFslgsjR52uz2AFbcOSzAAABA44WZ+eLdu3bRgwQKdd955MgxDL730kiZOnKjc3Fz179//tMc4HA7t3r3b+9xisQSq3FZj8UwAAALH1HDzox/9qNHzRx55RAsXLtT69evPGG4sFotSUlICUZ7PnGy5IdwAAOBvbabPjdvt1tKlS1VRUaHs7Owz7ldeXq6MjAylpaVp4sSJ2rFjR5Pv63K55HQ6Gz0CzcFoKQAAAsb0cLNt2zbFxMTIZrPp9ttv1/Lly9WvX7/T7tunTx/95S9/0VtvvaVXXnlFHo9HI0eO1IEDB874/vPnz1dcXJz3kZaW5q9TOSNvyw23pQAA8DuLYRimruZYU1Oj/Px8lZaWatmyZXrhhRe0Zs2aMwac76qtrVXfvn01depUPfzww6fdx+VyyeVyeZ87nU6lpaWptLRUDofDZ+fRlH/v+Vb/788blJkSq3/ddUlAPhMAgFDidDoVFxfXrO9vU/vcSFJkZKR69+4tSRo6dKg2btyop556SosWLTrrsRERERo8eLD27t17xn1sNptsNpvP6m0NWm4AAAgc029LfZ/H42nU0tIUt9utbdu2KTU11c9VnRvvDMUMBQcAwO9MbbmZO3eurrrqKqWnp6usrExLlixRTk6O3n//fUnStGnT1LVrV82fP1+SNG/ePI0YMUK9e/dWSUmJHn/8ceXl5em2224z8zTOqqFDcbmrTnVuj8LD2lymBAAgZJgabg4fPqxp06apsLBQcXFxysrK0vvvv68rrrhCkpSfny+r9WQQOH78uGbOnKmioiIlJCRo6NChWrt2bbP655gp1n7y11zuqlN8VKSJ1QAAENpM71AcaC3pkORL/e7/lypr3Pr4l5cpvWNUwD4XAIBQ0JLvb+6PBAgT+QEAEBiEmwBhCQYAAAKDcBMgtNwAABAYhJsAaRgxVUrLDQAAfkW4CRCHveG2FHPdAADgT4SbAPEunsltKQAA/IpwEyDeWYq5LQUAgF8RbgLkZIdibksBAOBPhJsAYSg4AACBQbgJEIaCAwAQGISbAPF2KGa0FAAAfkW4CRBabgAACAzCTYDQ5wYAgMAg3ARIQ8tNRY1btW6PydUAABC6CDcBEntihmJJKmM4OAAAfkO4CZDwMKuiI8MkcWsKAAB/ItwEUBxLMAAA4HeEmwBiODgAAP5HuAkghoMDAOB/hJsAYjg4AAD+R7gJIFpuAADwP8JNANHnBgAA/yPcBJDjxFw3tNwAAOA/hJsAami5KaXPDQAAfkO4CSBvnxvCDQAAfkO4CSDvaCmWXwAAwG8INwF0skMxLTcAAPgL4SaAGAoOAID/EW4CKI6h4AAA+B3hJoAaWm6qat2qqfOYXA0AAKGJcBNAMSfmuZGkMm5NAQDgF4SbAAqzWhRrY8QUAAD+RLgJMEZMAQDgX4SbAIs9cWuKWYoBAPAPwk2AeVtu6HMDAIBfEG4C7OQSDPS5AQDAHwg3ARZHyw0AAH5FuAkw7/pS9LkBAMAvCDcBxhIMAAD4F+EmwBwswQAAgF+1KtwUFBTowIED3ucbNmzQXXfdpeeff95nhYUqh71hEj9abgAA8IdWhZuf/OQn+uijjyRJRUVFuuKKK7Rhwwb95je/0bx583xaYKhhEj8AAPyrVeFm+/btuuiiiyRJr7/+ugYMGKC1a9fq1Vdf1eLFi5v9PgsXLlRWVpYcDoccDoeys7P13nvvNXnMG2+8oczMTNntdl1wwQV69913W3MKpjnZ54bbUgAA+EOrwk1tba1sNpsk6YMPPtB//Md/SJIyMzNVWFjY7Pfp1q2bFixYoM2bN2vTpk26/PLLNXHiRO3YseO0+69du1ZTp07VrbfeqtzcXE2aNEmTJk3S9u3bW3MapmgYLcUMxQAA+IfFMAyjpQcNHz5cl112mSZMmKBx48Zp/fr1GjhwoNavX69rr722UX+clkpMTNTjjz+uW2+99ZTXrr/+elVUVOidd97xbhsxYoQGDRqk5557rlnv73Q6FRcXp9LSUjkcjlbX2VoFxyp18WMfyRZu1e7fXhXwzwcAIBi15Pu7VS03jz76qBYtWqRLL71UU6dO1cCBAyVJ//jHP7y3q1rK7XZr6dKlqqioUHZ29mn3WbduncaOHdto2/jx47Vu3bozvq/L5ZLT6Wz0MFNDnxtXnUfVtW5TawEAIBSFt+agSy+9VEeOHJHT6VRCQoJ3+09/+lNFRUW16L22bdum7OxsVVdXKyYmRsuXL1e/fv1Ou29RUZGSk5MbbUtOTlZRUdEZ33/+/Pl66KGHWlSTP8XawmWxSIYhlVXXyR4RZnZJAACElFa13FRVVcnlcnmDTV5enn7/+99r9+7dSkpKatF79enTR1u2bNFnn32mO+64Q9OnT9fOnTtbU9ZpzZ07V6Wlpd5HQUGBz967NaxWi2JtDAcHAMBfWtVyM3HiRE2ePFm33367SkpKNHz4cEVEROjIkSN68skndccddzT7vSIjI9W7d29J0tChQ7Vx40Y99dRTWrRo0Sn7pqSkqLi4uNG24uJipaSknPH9bTabt/NzW+HoECFndR3DwQEA8INWtdx8/vnnuvjiiyVJy5YtU3JysvLy8vTXv/5VTz/99DkV5PF45HK5Tvtadna2Vq9e3WjbqlWrzthHp61iODgAAP7TqpabyspKxcbGSpJWrlypyZMny2q1asSIEcrLy2v2+8ydO1dXXXWV0tPTVVZWpiVLlignJ0fvv/++JGnatGnq2rWr5s+fL0maM2eORo8erSeeeEITJkzQ0qVLtWnTpqCbGZnFMwEA8J9Wtdz07t1bK1asUEFBgd5//32NGzdOknT48OEWDa8+fPiwpk2bpj59+mjMmDHauHGj3n//fV1xxRWSpPz8/Ebz5owcOVJLlizR888/r4EDB2rZsmVasWKFBgwY0JrTMA2LZwIA4D+tarm5//779ZOf/EQ///nPdfnll3tvC61cuVKDBw9u9vv8+c9/bvL1nJycU7Zdd911uu6661pUb1vD4pkAAPhPq8LNtddeqx/84AcqLCz0znEjSWPGjNE111zjs+JCVUPLDbMUAwDge60KN1L9yKWUlBTvbMTdunVr9QR+7Y23zw23pQAA8LlW9bnxeDyaN2+e4uLilJGRoYyMDMXHx+vhhx+Wx+PxdY0hx9vnhpYbAAB8rlUtN7/5zW/05z//WQsWLNCoUaMkSZ988okefPBBVVdX65FHHvFpkaEmrgNDwQEA8JdWhZuXXnpJL7zwgnc1cEnKyspS165ddeeddxJuzuJkh2JabgAA8LVW3ZY6duyYMjMzT9memZmpY8eOnXNRoc5hp88NAAD+0qpwM3DgQD3zzDOnbH/mmWeUlZV1zkWFOoaCAwDgP626LfXYY49pwoQJ+uCDD7xz3Kxbt04FBQV69913fVpgKPKGG1puAADwuVa13IwePVpfffWVrrnmGpWUlKikpESTJ0/Wjh079PLLL/u6xpDTcFuqps6j6lq3ydUAABBaLIZhGL56s61bt2rIkCFyu9vuF7bT6VRcXJxKS0tbtFSEL3k8hnr/5l15DGnDr8coyWE3pQ4AAIJFS76/W9Vyg3NjtVqUfCLQ5B2rNLkaAABCC+HGJFnd4iRJWwtKzC0EAIAQQ7gxycC0eElSLuEGAACfatFoqcmTJzf5eklJybnU0q4M6hYviZYbAAB8rUXhJi4u7qyvT5s27ZwKai8u6BYni0U6cLxKR8pd6hRjM7skAABCQovCzYsvvuivOtqdWHuEeneO0Z7D5dpaUKIxfZPNLgkAgJBAnxsTNfS74dYUAAC+Q7gx0aAT4WbLgVJzCwEAIIQQbkw06DstNz6cSxEAgHaNcGOiPimxsoVbVVpVq/1HmcwPAABfINyYKCLMqgFdmcwPAABfItyYbOCJ+W62EG4AAPAJwo3JBqbVt9wQbgAA8A3CjckGpyVIknYecqqmzmNyNQAABD/CjcnSEjsoISpCNW6PdhU6zS4HAICgR7gxmcViOTmZ34ESU2sBACAUEG7aAO9kfvS7AQDgnBFu2oCBhBsAAHyGcNMGNAwH/+bbCpVW1ZpbDAAAQY5w0wYkRkcqo2OUJGkb60wBAHBOCDdtxMnJ/I6bWwgAAEGOcNNGnOx3Q8sNAADngnDTRnx3xBQrhAMA0HqEmzaifxeHwq0WHSl36VBptdnlAAAQtAg3bYQ9IkyZqbGSWCEcAIBzQbhpQ5jMDwCAc0e4aUNOjpgqMbUOAACCGeGmDWloudl2oFR1blYIBwCgNQg3bUivzjGKsYWrqtatPYfLzS4HAICgRLhpQ6xWi7K6xUmiUzEAAK1FuGljGibz23qgxNQ6AAAIVqaGm/nz52vYsGGKjY1VUlKSJk2apN27dzd5zOLFi2WxWBo97HZ7gCr2v4Z+N7n5JabWAQBAsDI13KxZs0azZs3S+vXrtWrVKtXW1mrcuHGqqKho8jiHw6HCwkLvIy8vL0AV+19DuPmquEyVNXXmFgMAQBAKN/PD//WvfzV6vnjxYiUlJWnz5s265JJLznicxWJRSkqKv8szRbLDrhSHXUXOam0/6NRFPRLNLgkAgKDSpvrclJbWLxqZmNj0F3p5ebkyMjKUlpamiRMnaseOHWfc1+Vyyel0Nnq0dScn82OFcAAAWqrNhBuPx6O77rpLo0aN0oABA864X58+ffSXv/xFb731ll555RV5PB6NHDlSBw4cOO3+8+fPV1xcnPeRlpbmr1PwGW+nYlYIBwCgxSxGG1mC+o477tB7772nTz75RN26dWv2cbW1terbt6+mTp2qhx9++JTXXS6XXC6X97nT6VRaWppKS0vlcDh8Uruvrf36iH7yp8/UNb6DPv3V5WaXAwCA6ZxOp+Li4pr1/W1qn5sGs2fP1jvvvKOPP/64RcFGkiIiIjR48GDt3bv3tK/bbDbZbDZflBkwWd3iZbFIB0uq9G2ZS51jg6t+AADMZOptKcMwNHv2bC1fvlwffvihevTo0eL3cLvd2rZtm1JTU/1QoTlibOE6LylGEpP5AQDQUqaGm1mzZumVV17RkiVLFBsbq6KiIhUVFamqqsq7z7Rp0zR37lzv83nz5mnlypX65ptv9Pnnn+umm25SXl6ebrvtNjNOwW8aFtFkMj8AAFrG1HCzcOFClZaW6tJLL1Vqaqr38be//c27T35+vgoLC73Pjx8/rpkzZ6pv3766+uqr5XQ6tXbtWvXr18+MU/CbQenxklghHACAlmozHYoDpSUdksy0/WCpfviHT+Swh2vL/eNktVrMLgkAANO05Pu7zQwFR2N9UmJlC7fKWV2nfUebnrEZAACcRLhpoyLCrLqgKyuEAwDQUoSbNuzkZH4lptYBAEAwIdy0YSeXYSgxtQ4AAIIJ4aYNawg3OwudctW5zS0GAIAgQbhpw7oldFBidKRq3YZ2FZaZXQ4AAEGBcNOGWSyWk7em8lkhHACA5iDctHEnZypmhXAAAJqDcNPGDUxjODgAAC1BuGnjGm5LfXOkQqWVteYWAwBAECDctHHxUZHq3jFKkrSFRTQBADgrwk0QGJKRIElavavY5EoAAGj7CDdB4JrBXSVJK3IPqrqW+W4AAGgK4SYIjOzVSV3jO8hZXaf3dxSZXQ4AAG0a4SYIhFktmjK0myTpjU0HTK4GAIC2jXATJK47EW4+2XtEBccqTa4GAIC2i3ATJNISozSqd0dJ0rLNtN4AAHAmhJsg8uML0yTVhxuPxzC5GgAA2ibCTRAZ3z9FDnu4DpZU6dOvj5hdDgAAbRLhJojYI8I0cVD9sPDX6VgMAMBpEW6CTMOtqfd3FKmkssbkagAAaHsIN0FmQFeHMlNiVVPn0T+2HjK7HAAA2hzCTZCxWCy6flh9683fNhaYXA0AAG0P4SYITRrUVZFhVu045NT2g6VmlwMAQJtCuAlCCdGRuqJfsiTmvAEA4PsIN0HqxyduTS1nMU0AABoh3ASpH/TupNQ4u0qrarVqZ7HZ5QAA0GYQboJUmNWia0+sN/X6JjoWAwDQgHATxK4bWn9r6pO9R3TgOItpAgAgEW6CWnrHKGX37CjDkP6++aDZ5QAA0CYQboLcj4fV35p6Y3MBi2kCACDCTdC7sn+qYm3hOnC8Suu+OWp2OQAAmI5wE+Q6RIbpPwZ1kUTHYgAAJMJNSGhYTPO97UUqraw1uRoAAMxFuAkBWd3i1Cf5xGKaX7CYJgCgfSPchACLxeKdsfh1FtMEALRzhJsQMWlQF0WEWbTtYKl2HnKaXQ4AAKYh3ISIjjE2je1bv5jmG5tpvQEAtF+EmxDy3cU0XXUspgkAaJ8INyHkkvM6K8VhV0llrT7YedjscgAAMAXhJoSEWS2aMrSrJOmZj/aqps5jckUAAAQe4SbEzBjZQwlREdpV6NTTq/eYXQ4AAAFnariZP3++hg0bptjYWCUlJWnSpEnavXv3WY974403lJmZKbvdrgsuuEDvvvtuAKoNDp1jbfrtpAskSc/m7FVu/nGTKwIAILBMDTdr1qzRrFmztH79eq1atUq1tbUaN26cKioqznjM2rVrNXXqVN16663Kzc3VpEmTNGnSJG3fvj2AlbdtE7JSNXFQF3kM6Z7Xt6qqhs7FAID2w2IYRptZSvrbb79VUlKS1qxZo0suueS0+1x//fWqqKjQO++84902YsQIDRo0SM8999xZP8PpdCouLk6lpaVyOBw+q72tKa2s1bjfr1Gx06UZI7vrwf/ob3ZJAAC0Wku+v9tUn5vS0lJJUmJi4hn3WbduncaOHdto2/jx47Vu3brT7u9yueR0Ohs92oO4qAg9OiVLkrR47X6t3XvE5IoAAAiMNhNuPB6P7rrrLo0aNUoDBgw4435FRUVKTk5utC05OVlFRUWn3X/+/PmKi4vzPtLS0nxad1t2aZ8k/WR4uiTpl8u+kLOaRTUBAKGvzYSbWbNmafv27Vq6dKlP33fu3LkqLS31PgoK2tfsvb+5uq/SE6N0sKRKD7+90+xyAADwuzYRbmbPnq133nlHH330kbp169bkvikpKSouLm60rbi4WCkpKafd32azyeFwNHq0J9G2cP3fdQNlsUhvbD6gD3YWn/0gAACCmKnhxjAMzZ49W8uXL9eHH36oHj16nPWY7OxsrV69utG2VatWKTs7219lBr2LeiRq5sU9JUm/enObjlXUmFwRAAD+Y2q4mTVrll555RUtWbJEsbGxKioqUlFRkaqqqrz7TJs2TXPnzvU+nzNnjv71r3/piSee0JdffqkHH3xQmzZt0uzZs804haBx9xXn6/zkGB0pd+l/VmxTGxokBwCAT5kabhYuXKjS0lJdeumlSk1N9T7+9re/effJz89XYWGh9/nIkSO1ZMkSPf/88xo4cKCWLVumFStWNNkJGZI9IkxP/niQwq0WvbutSP/YesjskgAA8Is2Nc9NILSXeW7O5KkP9uh3H3wlhz1cK38+WilxdrNLAgDgrIJ2nhv4352X9VJWtzg5q+t079+/4PYUACDkEG7amYgwq5788UBFhlu15qtvtWRDvtklAQDgU4Sbdqh3Uqz+e3wfSdIj/9ylvKNnXssLAIBgQ7hpp24Z1UPDeySqssat/3otVyWVDA8HAIQGwk07ZbVa9H/XDZTDHq6tB0o1eeFa5R+tNLssAADOGeGmHUtLjNLrt2erS5xd33xboWue/VS5+cfNLgsAgHNCuGnnMlMcWj5rlPp3cehoRY1ueH69/rX99IuQAgAQDAg3ULLDrtd/lq3L+nSWq86jO17drBf+/Q3DxAEAQYlwA0n1C2z+adqFunF4ugxD+u0/d+mht3fK7SHgAACCC+EGXuFhVv120gD9+upMSdLitfv1s5c3qbKmzuTKAABoPsINGrFYLPrpJb307I1DZAu36oNdh3X9ovU6XFZtdmkAADQL4QandfUFqVoyc4QSoyO17WCprvnjWn1VXGZ2WQAAnBXhBmc0NCNBy+8cqR6donWwpEpTFq7V2r1HzC4LAIAmEW7QpIyO0XrzjpEa1j1BZdV1uunPn+nu17ewZAMAoM0i3OCsEqIj9fKtwzV5SFd5DOnNzw/q8ifW6N5lX+jAcWY1BgC0LRajnU1m4nQ6FRcXp9LSUjkcDrPLCTpfHCjRk6u+Us7ubyVJEWEW/fjCNM2+vLdS4zqYXB0AIFS15PubcINW2Zx3XL9b9ZU+OdEHJzLMqp8MT9edl/ZSksNucnUAgFBDuGkC4ca3PvvmqJ5c9ZU+23dMkmQLt+r/jcjQ7Zf2UqcYm8nVAQBCBeGmCYQb3zMMQ+u+PqonVn2lzXn1C292iAjT9JHd9bNLeiohOtLkCgEAwY5w0wTCjf8YhqGP9xzRkyt3a+uBUklSrC1cPxvdUzeP6qFoW7jJFQIAghXhpgmEG/8zDEOrdx3WE6u+0q5CpySpU0yk/vPy8zT1onRFhjNIDwDQMoSbJhBuAsfjMfT2F4f05KqvlHe0fsh4t4QOuvuK8zVxUFeFWS0mVwgACBaEmyYQbgKv1u3R3zYW6KnVe/RtmUuS1Cc5Vr8Y30dj+ybJYiHkAACaRrhpAuHGPFU1bi1eu18Lc/bKWV2/0viQ9Hj995WZGtGzo8nVAQDaMsJNEwg35iutrNWij7/WXz7dp+pajyRp9PmdNfvy3rowI4GWHADAKQg3TSDctB2HndX6w4d79dqGfNV56v817NU5WjcMS9fkIV3VkXlyAAAnEG6aQLhpe/KOVuiPH+3V21sLVVXrllS/rMO4fim6fliaftC7k6x0PgaAdo1w0wTCTdtVVl2rt7cWaunGfH1xYp4cSeoa30HXD0vTdRd2Y/0qAGinCDdNINwEhx2HSvX6xgItzz3o7XxstUiX9knS9cPSdHlmkiLCmC8HANoLwk0TCDfBpbrWrfe2F2rphgLv+lVS/czHg9LjNTQjQRdmJGpQerximAEZAEIW4aYJhJvg9c235frbpgL9ffMBHSmvafSa1SL1SXHowowEDT3x6JbQgZFXABAiCDdNINwEP7fH0O6iMm3OO6bNece1Ke+4DhyvOmW/pFibLuyeoIu6J+qHA7uwSjkABDHCTRMIN6Gp2FmtzXnHvWFnx8FS7/By6cToq/4p+slF6cru2ZHRVwAQZAg3TSDctA/VtW5tLSjRprzjWrmjyLtKuSSlJ0bphovSdO3QbkqKtZtYJQCguQg3TSDctE/bD5Zq6cZ8rcg9pHJX/eircKtFV/RL1tSL0plLBwDaOMJNEwg37VtlTZ3e+aJQr23IV25+iXd7WmIH3TAsXdcN7aYkB605ANDWEG6aQLhBg12FTi3dkK83cw+q7MRcOmFWi8b3T9aMkT00rDvrXAFAW0G4aQLhBt9XVePWP7cVaumGfG3KO+7d3jfVoRkjMzRxUFfZI8JMrBAAQLhpAuEGTdlV6NRf1+3X8tyD3hXL46MidP2wNP2/ERnqlhDV4vd0Vtdq5yGnauo86tfFwZB0AGgFwk0TCDdojpLKGr2+qUB/XZfnnUPHapHG9k3WjJHdld2r42lvWR0uq9aOg07tOFSqHYec2nHIqfxjlY32SXHYNaCrQ/27xGlA1zgN6OpQisPOLTAAaALhpgmEG7SE22No9a5ivbRuvz7de9S7/fzkGE3L7q7E6MhGQebbMtdp36drfAfZwq3ad7RCp/sb1zE6Uv27xmlAF4cGdI3T+cmxiraFKSLMqogwqyLDrIoIsyjMaiEEAWiXgibcfPzxx3r88ce1efNmFRYWavny5Zo0adIZ98/JydFll112yvbCwkKlpKQ06zMJN2itPcVlemndfr35+UFV1rhPu4/FIvXsFK3+XeLU/0RQ6ZfqUEJ0pCSp3FWnXYVObT9Yqu0nWnj2HC6X29O8v4YWixqFnYgwq+wRYRrfP1mzLuut+KhIn50vALQlLfn+NnWlwYqKCg0cOFC33HKLJk+e3Ozjdu/e3ejEkpKS/FEe0Mh5ybH67aQL9MvxmVq2+YCWbT6gMKvUPzVO/bs61L+LQ5kpDkU3sYBnjC1cw7onalj3RO+26lq3dheVafuhk4Fn37cVctV5VOP2NDreMKSaOo9q6hpv/9O/9+lvGws0+/LempbdnQ7QANq1NnNbymKxNLvl5vjx44qPj2/V59Byg2BiGIbqPIZq3R7V1hmqcXvqfz7xqKkzlH+sUr//4Ct9WVQmqf4W2D3jztekQV3PaWLCWrdHVbVuOewRvjodAGi1oGm5aa1BgwbJ5XJpwIABevDBBzVq1Kgz7utyueRynewH4XQ6A1Ei4BMWi8V7+0lnuOPUr4tDV/RL1pufH9CTq77SwZIq3f36Vr3w732ae3WmLj6vc7M/73hFjXK+OqwPdh7Wx199qzJXnXp2itbQjARd2D1BQzMS1atzNP1+ALRpQdVys3v3buXk5OjCCy+Uy+XSCy+8oJdfflmfffaZhgwZctpjHnzwQT300EOnbKflBqGoutatFz/dr2c/2quyE8tMXHxeJ/3qqkz17xJ32mO+/rZcq3cV64Odh7Up75jO1v0nISriRNhJ1IUZCRrQNY7bYAD8Lmg6FH9Xc8LN6YwePVrp6el6+eWXT/v66Vpu0tLSCDcIaccqavTMh3v18vr9qnUbslikawZ11d3jzleKw67Necf1wa5ird51WN8cqWh0bGZKrMb2TdaYvknq3jFauQXHtWl//WrrWwtK5Ppef5/IMKsu6BanCzMS1P9EB+oenaIVxlpdAHwo5G9LfddFF12kTz755Iyv22w22WxMmob2JTE6Uvf/qJ9mjOyux1fu1ttbD+nN3IN6Z1uhoiLDVFJZ6903IsyiET07egPN9ycqvDwzWZdnJkuq78y841CpNuc1BJ5jOlJeo815x7X5O7M72yOs6pPiUL9Uh/p1qf8zMyW2yc7WZ2IYhnf4vMUibokBOKugDzdbtmxRamqq2WUAbVJ6xyj9Yepgzby4h/733V1a/80x1dR5lBAVocv6JGlsv2RdfF4nxTaz03BkuFWD0xM0OD1Bt11cHzzyjlZqU95x5eYf185Cp74sLFNVrVtbC0q0taDEe6zFInXvGK1+qQ71TY1VRJhV5a66+kd1nSpq6lRWXaeKE9sqXG6VVdeqosZ92qHyFotkUX3YsXifWxRtC9PQjERl9+qoET0T1TfFwYrvQDtj6m2p8vJy7d27V5I0ePBgPfnkk7rsssuUmJio9PR0zZ07VwcPHtRf//pXSdLvf/979ejRQ/3791d1dbVeeOEF/eEPf9DKlSs1ZsyYZn0mo6XQXhmGoS0FJXJ7DA1Ki1d4mNUvn+P2GMo7WqGdhU7tPOTUzkKndhU6Vew8/QSH/hYfFaHhPRKV3bOjsnt10vnJMbT+AEEoaG5Lbdq0qdGkfHfffbckafr06Vq8eLEKCwuVn5/vfb2mpkb33HOPDh48qKioKGVlZemDDz447cR+ABqzWCwanJ7g988Js1rUs3OMenaO0Q+zuni3Hyl3adeJwLO7uH7YeowtXDG2cEXbwhVrD1d0ZLhi7OGKPbEtxl7/ekSYtf72lOrn+jFk6MQ/3ufGieeHndVa/80xrf/mqDbuP6aSylq9v6NY7+8ollQ/G/Twng1hp6N6dorxecuOYRhy1Xnk9tQP5a9zn/z55J+eE68ZskdY1aszoQvwlTbToThQaLkB2o9at0dfHCjV+m+Oav03R7Vp/3FV1TaeXTrGFu7tG9Qwq3TvpJj64ffNUF3r1p7i8vrgdqKValehU87quhbV2jspRjcMS9PkId2UGM1M08D3BeVoqUAh3ADtV02dR18cKNG6r49q3TdHtTnv+Cmjv6T6vkV9kmPVv4tD/bvWL6XRN8XhXT7ju0Hm628rmrV8htUihVutCrNaFG61KCzMonCrVeFWi45X1njriAizaFz/FE0dlq6RvTrSXwg4gXDTBMINgAZ1bo++/rbCu/jp9oOl2lnoVFkLW13ioyLUN6W+9afviQ7TaYlRigyrDzNhFkuTIaWsulb/2HpISzcUaNvBUu/29MQoXT8sTdcN7aYkh73V52m2OrcnKBZ9LXfVKf9opfKPVajgWJUMGfW3R0+5bRqhaFuYYuzhsoUzx1OgEG6aQLgB0BTDMFRwrErbD5V+J/Q4daTcJYtF6tEx2htg+p64nZXisPvsi3v7wVIt3Zivt3IPeSdiDLNadFmfJE29KE2jz+/cqDO4YRiqqHGrvLpOZdW1clbXjzYrq65VeXWdLBYp5sSXcaw93Ptl3dDXqbm335qrYQTd5/nH6x95JfqyyKmkWLtG9e6kUb07alTvTko2Iax5PIa+LXcp72il8o9VKv9ohfKONfxcqaMVNS1+z8gwq6JtYUp22DU4PUFDM+of3TtGtfkwF2wIN00g3ABojSPlLnWICGvVXD2tUVXj1j+3FWrphnxt+s4cQkmxNiVGR6rsRJgpd9WddVbpptjCrfVhxx6u+A4RSomzKzWug7rEN/4zKdZ22hF2lTV12lpQqs/z66cDyM0vaVZI6J0Uox/07qSRvTpqRK+OLVrDzDAMlVbVqtjp0rdlLjmra+WsqpWzulalVbVyVtV5t5VW1Qc+Z1WtSqpqT1l09vsSoiKU3jFa6YlRirBaVPadqQrKTwTHcledKmvcTb5PYnSkhqTHa0hGgoakJ2hgt3h1iGx5K49hGEEXkmrdHlW63IqL8u26dISbJhBuAASbPcVlWrqxQG9+fkDHvzMB43eFWS2KtdffPom1RXhHnRlSo7mEGr6gT9fXqClWi5TssCs1zq7U+A6KtYVr28FSfVlUdkqfo8gwqwZ0dWhIeoKGZCTogq5xyjtaqU/2HtHar49o28FSffebx2qRsrrFe1t1UuM6qNhZ/Z2HS0XOah0+8XOxs7rF9X/399Ql3q6MxGilJUYpo2OUMhKjlJYYpfSOUc0OWW6P0ej3ue9IhT4/MZnlFwdLTwlR4VaL+nWp/530TY2Vq86jsur6EFZ2Inw1BNb6P+tfq6p1q3vHaA1Ki9fg9HgNSotXZopDkeH+mcrhXHxb5tJrG/L16md5ujwzWfMnX+DT9yfcNIFwAyBYuerc2rDvmCQp1h6hGFu4HPZwxdojZI+wtuj/8GvdHlW4TkyceOJL+lhFjQpLq3WotEqFJdUqLK3SoZL6gFHXRPNQapxdQ9ITNPhES0X/Lo4m+6KUVNZo/TdH68PO3qOnLAHSXAlREeoUY1N8VIQc9gg5OkQorkOEHPZwOTrUP6/fHn5ie33LlK9vxX3fd2fyzs0v0aa8Yz6d58kWbtWArnEanBavQenxGpyeoC5xvrs12lJbCkr00tr9+ucXhapx14e69MQofXjPaJ/Op0W4aQLhBgBaxu0xdKTcpUMlVfXhp6RKpVW16pMSqyHpCeoS3+Gc3v9QSZU+3XtEa78+qk/3HlFZdZ1S4uxKirUpJc6uZEfDw6Zkh10pDrs6x9qCZsFWwzB0qLTa27LzzZEKRUeGnWhpi1CsPVyOE3/G2iO8gTXWHi5bhFW7i8qUm1+iLQX1j9KqU1vvOsfaNDgtXhkdoxQVGa5oW5g6RIYrOjJMUZFhiooMb/ynLUxxHSJa3SHaVefWu9sKtXhtXqOZyAelxWvGyO66+oJUn7cuEW6aQLgBAAQrwzC070iFN+zkFhzXl4VlTbasnYn1xJIo5yfH6vyUWPVJjlWflBhldIw+Y+tWsbNar67P05IN+TpSXt+3KjLMqh9mpWr6yO4amBZ/LqfXJMJNEwg3AIBQUlXj1vZDpdpaUKLDZS5VuOpUVeNWRU19x+eTjxPPXXWqrHXrTN/+kWFW9ewcrT4psTo/uT70dIgM02sb8vWv7UXeIJXisOumEem64aJ0dYrx/wLVhJsmEG4AAO2dYdQPi99TXK4vi8r0VVGZdheXaU9xmSrOMhLsou6Jmj6yu8b1T/Z7/6XvCpq1pQAAQOBZLBYlxdq98w818HgMHSyp0lfF9WGnPvSU69uyao3JTNb0kd3Vr0vbbxgg3AAAAEmS1WpR2omh8WP6JptdTqu1vYHyAAAA54BwAwAAQgrhBgAAhBTCDQAACCmEGwAAEFIINwAAIKQQbgAAQEgh3AAAgJBCuAEAACGFcAMAAEIK4QYAAIQUwg0AAAgphBsAABBSCDcAACCkhJtdQKAZhiFJcjqdJlcCAACaq+F7u+F7vCntLtyUlZVJktLS0kyuBAAAtFRZWZni4uKa3MdiNCcChRCPx6NDhw4pNjZWFovFp+/tdDqVlpamgoICORwOn753WxDq5yeF/jlyfsEv1M+R8wt+/jpHwzBUVlamLl26yGptuldNu2u5sVqt6tatm18/w+FwhOy/tFLon58U+ufI+QW/UD9Hzi/4+eMcz9Zi04AOxQAAIKQQbgAAQEgh3PiQzWbTAw88IJvNZnYpfhHq5yeF/jlyfsEv1M+R8wt+beEc212HYgAAENpouQEAACGFcAMAAEIK4QYAAIQUwg0AAAgphBsf+eMf/6ju3bvLbrdr+PDh2rBhg9kl+cyDDz4oi8XS6JGZmWl2Wa328ccf60c/+pG6dOkii8WiFStWNHrdMAzdf//9Sk1NVYcOHTR27Fjt2bPHnGJb6WznOGPGjFOu6ZVXXmlOsa0wf/58DRs2TLGxsUpKStKkSZO0e/fuRvtUV1dr1qxZ6tixo2JiYjRlyhQVFxebVHHLNOf8Lr300lOu4e23325SxS2zcOFCZWVleSd5y87O1nvvved9PZivXYOznWMwX7/TWbBggSwWi+666y7vNjOvI+HGB/72t7/p7rvv1gMPPKDPP/9cAwcO1Pjx43X48GGzS/OZ/v37q7Cw0Pv45JNPzC6p1SoqKjRw4ED98Y9/PO3rjz32mJ5++mk999xz+uyzzxQdHa3x48eruro6wJW23tnOUZKuvPLKRtf0tddeC2CF52bNmjWaNWuW1q9fr1WrVqm2tlbjxo1TRUWFd5+f//znevvtt/XGG29ozZo1OnTokCZPnmxi1c3XnPOTpJkzZza6ho899phJFbdMt27dtGDBAm3evFmbNm3S5ZdfrokTJ2rHjh2SgvvaNTjbOUrBe/2+b+PGjVq0aJGysrIabTf1Oho4ZxdddJExa9Ys73O322106dLFmD9/volV+c4DDzxgDBw40Owy/EKSsXz5cu9zj8djpKSkGI8//rh3W0lJiWGz2YzXXnvNhArP3ffP0TAMY/r06cbEiRNNqccfDh8+bEgy1qxZYxhG/TWLiIgw3njjDe8+u3btMiQZ69atM6vMVvv++RmGYYwePdqYM2eOeUX5WEJCgvHCCy+E3LX7roZzNIzQuX5lZWXGeeedZ6xatarROZl9HWm5OUc1NTXavHmzxo4d691mtVo1duxYrVu3zsTKfGvPnj3q0qWLevbsqRtvvFH5+flml+QX+/btU1FRUaPrGRcXp+HDh4fU9ZSknJwcJSUlqU+fPrrjjjt09OhRs0tqtdLSUklSYmKiJGnz5s2qra1tdB0zMzOVnp4elNfx++fX4NVXX1WnTp00YMAAzZ07V5WVlWaUd07cbreWLl2qiooKZWdnh9y1k049xwahcP1mzZqlCRMmNLpekvl/B9vdwpm+duTIEbndbiUnJzfanpycrC+//NKkqnxr+PDhWrx4sfr06aPCwkI99NBDuvjii7V9+3bFxsaaXZ5PFRUVSdJpr2fDa6Hgyiuv1OTJk9WjRw99/fXX+vWvf62rrrpK69atU1hYmNnltYjH49Fdd92lUaNGacCAAZLqr2NkZKTi4+Mb7RuM1/F05ydJP/nJT5SRkaEuXbroiy++0L333qvdu3frzTffNLHa5tu2bZuys7NVXV2tmJgYLV++XP369dOWLVtC5tqd6Ryl4L9+krR06VJ9/vnn2rhx4ymvmf13kHCDs7rqqqu8P2dlZWn48OHKyMjQ66+/rltvvdXEytBaN9xwg/fnCy64QFlZWerVq5dycnI0ZswYEytruVmzZmn79u1B3Q+sKWc6v5/+9Kfeny+44AKlpqZqzJgx+vrrr9WrV69Al9liffr00ZYtW1RaWqply5Zp+vTpWrNmjdll+dSZzrFfv35Bf/0KCgo0Z84crVq1Sna73exyTsFtqXPUqVMnhYWFndIDvLi4WCkpKSZV5V/x8fE6//zztXfvXrNL8bmGa9aerqck9ezZU506dQq6azp79my98847+uijj9StWzfv9pSUFNXU1KikpKTR/sF2Hc90fqczfPhwSQqaaxgZGanevXtr6NChmj9/vgYOHKinnnoqZK6ddOZzPJ1gu36bN2/W4cOHNWTIEIWHhys8PFxr1qzR008/rfDwcCUnJ5t6HQk35ygyMlJDhw7V6tWrvds8Ho9Wr17d6N5qKCkvL9fXX3+t1NRUs0vxuR49eiglJaXR9XQ6nfrss89C9npK0oEDB3T06NGguaaGYWj27Nlavny5PvzwQ/Xo0aPR60OHDlVERESj67h7927l5+cHxXU82/mdzpYtWyQpaK7h93k8HrlcrqC/dk1pOMfTCbbrN2bMGG3btk1btmzxPi688ELdeOON3p9NvY5+77LcDixdutSw2WzG4sWLjZ07dxo//elPjfj4eKOoqMjs0nzinnvuMXJycox9+/YZn376qTF27FijU6dOxuHDh80urVXKysqM3NxcIzc315BkPPnkk0Zubq6Rl5dnGIZhLFiwwIiPjzfeeust44svvjAmTpxo9OjRw6iqqjK58uZr6hzLysqMX/ziF8a6deuMffv2GR988IExZMgQ47zzzjOqq6vNLr1Z7rjjDiMuLs7IyckxCgsLvY/KykrvPrfffruRnp5ufPjhh8amTZuM7OxsIzs728Sqm+9s57d3715j3rx5xqZNm4x9+/YZb731ltGzZ0/jkksuMbny5vnVr35lrFmzxti3b5/xxRdfGL/61a8Mi8VirFy50jCM4L52DZo6x2C/fmfy/RFgZl5Hwo2P/OEPfzDS09ONyMhI46KLLjLWr19vdkk+c/311xupqalGZGSk0bVrV+P666839u7da3ZZrfbRRx8Zkk55TJ8+3TCM+uHg9913n5GcnGzYbDZjzJgxxu7du80tuoWaOsfKykpj3LhxRufOnY2IiAgjIyPDmDlzZlCF8dOdmyTjxRdf9O5TVVVl3HnnnUZCQoIRFRVlXHPNNUZhYaF5RbfA2c4vPz/fuOSSS4zExETDZrMZvXv3Nn75y18apaWl5hbeTLfccouRkZFhREZGGp07dzbGjBnjDTaGEdzXrkFT5xjs1+9Mvh9uzLyOFsMwDP+3DwEAAAQGfW4AAEBIIdwAAICQQrgBAAAhhXADAABCCuEGAACEFMINAAAIKYQbAAAQUgg3AAAgpBBuALRJ3377re644w6lp6fLZrMpJSVF48eP16effipJslgsWrFihblFAmiTws0uAABOZ8qUKaqpqdFLL72knj17qri4WKtXr9bRo0fNLg1AG8fyCwDanJKSEiUkJCgnJ0ejR48+5fXu3bsrLy/P+zwjI0P79++XJL311lt66KGHtHPnTnXp0kXTp0/Xb37zG4WH1/+/nMVi0bPPPqt//OMfysnJUWpqqh577DFde+21ATk3AP7HbSkAbU5MTIxiYmK0YsUKuVyuU17fuHGjJOnFF19UYWGh9/m///1vTZs2TXPmzNHOnTu1aNEiLV68WI888kij4++77z5NmTJFW7du1Y033qgbbrhBu3bt8v+JAQgIWm4AtEl///vfNXPmTFVVVWnIkCEaPXq0brjhBmVlZUmqb4FZvny5Jk2a5D1m7NixGjNmjObOnevd9sorr+i///u/dejQIe9xt99+uxYuXOjdZ8SIERoyZIieffbZwJwcAL+i5QZAmzRlyhQdOnRI//jHP3TllVcqJydHQ4YM0eLFi894zNatWzVv3jxvy09MTIxmzpypwsJCVVZWevfLzs5udFx2djYtN0AIoUMxgDbLbrfriiuu0BVXXKH77rtPt912mx544AHNmDHjtPuXl5froYce0uTJk0/7XgDaB1puAASNfv36qaKiQpIUEREht9vd6PUhQ4Zo9+7d6t279ykPq/Xkf+7Wr1/f6Lj169erb9++/j8BAAFByw2ANufo0aO67rrrdMsttygrK0uxsbHatGmTHnvsMU2cOFFS/Yip1atXa9SoUbLZbEpISND999+vH/7wh0pPT9e1114rq9WqrVu3avv27frtb3/rff833nhDF154oX7wgx/o1Vdf1YYNG/TnP//ZrNMF4GN0KAbQ5rhcLj344INauXKlvv76a9XW1iotLU3XXXedfv3rX6tDhw56++23dffdd2v//v3q2rWrdyj4+++/r3nz5ik3N1cRERHKzMzUbbfdppkzZ0qq71D8xz/+UStWrNDHH3+s1NRUPfroo/rxj39s4hkD8CXCDYB25XSjrACEFvrcAACAkEK4AQAAIYUOxQDaFe7EA6GPlhsAABBSCDcAACCkEG4AAEBIIdwAAICQQrgBAAAhhXADAABCCuEGAACEFMINAAAIKYQbAAAQUv4/4xPDxD+s8CYAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAASRhJREFUeJzt3Xl4VPXd///XZJuQZSYhkAUIYd8FZA98FVoQUOoNbq0Uf6BWWxXuG7rdLW1VxHqDenu3Wltc2kpVLFUqUK0oiAY3QHZZI3sC2diSyTpJZs7vjySjkRCSMDNnMnk+rutcyZw5Z/I+OZW8+lnOx2IYhiEAAIAgEWJ2AQAAAN5EuAEAAEGFcAMAAIIK4QYAAAQVwg0AAAgqhBsAABBUCDcAACCoEG4AAEBQIdwAAICgQrgB4HN33nmnunXr1qJzFy1aJIvF4t2CAAQ1wg3QhlksliZtGRkZZpdqijvvvFMxMTFmlwGgmSysLQW0Xa+++mq91y+//LI2bNigV155pd7+6667TklJSS3+OVVVVXK73bJarc0+t7q6WtXV1YqMjGzxz2+pO++8U6tWrVJJSYnffzaAlgszuwAA5rnjjjvqvd6yZYs2bNhw0f5vKisrU1RUVJN/Tnh4eIvqk6SwsDCFhfFPFYCmo1sKQKMmTJigQYMGaceOHbr22msVFRWlX/3qV5KktWvXatq0aerUqZOsVqt69uypRx99VC6Xq95nfHPMzYkTJ2SxWPS///u/euGFF9SzZ09ZrVaNHDlS27Ztq3duQ2NuLBaL5s2bpzVr1mjQoEGyWq0aOHCg3n333Yvqz8jI0IgRIxQZGamePXvq+eef9/o4njfeeEPDhw9Xu3bt1KFDB91xxx06ffp0vWPy8vJ01113qUuXLrJarUpJSdH06dN14sQJzzHbt2/XlClT1KFDB7Vr107du3fX3Xff7bU6gbaC/zsE4LLOnTun66+/XrfffrvuuOMOTxfV8uXLFRMTo5/85CeKiYnRBx98oIceekgOh0NPPvnkZT/3tddeU3FxsX70ox/JYrHoiSee0M0336xjx45dtrXnk08+0ZtvvqkHHnhAsbGxeuaZZ3TLLbcoKytLCQkJkqRdu3Zp6tSpSklJ0SOPPCKXy6XFixerY8eOV/5LqbV8+XLdddddGjlypJYsWaL8/Hw9/fTT+vTTT7Vr1y7FxcVJkm655Rbt379f//mf/6lu3bqpoKBAGzZsUFZWluf15MmT1bFjR/3yl79UXFycTpw4oTfffNNrtQJthgEAtebOnWt885+F8ePHG5KM55577qLjy8rKLtr3ox/9yIiKijIqKio8++bMmWOkpaV5Xh8/ftyQZCQkJBjnz5/37F+7dq0hyXjrrbc8+x5++OGLapJkREREGEeOHPHs27NnjyHJ+MMf/uDZd+ONNxpRUVHG6dOnPfsOHz5shIWFXfSZDZkzZ44RHR19yfcrKyuNxMREY9CgQUZ5ebln/9tvv21IMh566CHDMAzjwoULhiTjySefvORnrV692pBkbNu27bJ1AWgc3VIALstqtequu+66aH+7du083xcXF+vs2bO65pprVFZWpkOHDl32c7/3ve8pPj7e8/qaa66RJB07duyy506aNEk9e/b0vB48eLBsNpvnXJfLpffff18zZsxQp06dPMf16tVL119//WU/vym2b9+ugoICPfDAA/UGPE+bNk39+vXTv//9b0k1v6eIiAhlZGTowoULDX5WXQvP22+/raqqKq/UB7RVhBsAl9W5c2dFRERctH///v266aabZLfbZbPZ1LFjR89g5KKiost+bteuXeu9rgs6lwoAjZ1bd37duQUFBSovL1evXr0uOq6hfS1x8uRJSVLfvn0veq9fv36e961Wqx5//HGtW7dOSUlJuvbaa/XEE08oLy/Pc/z48eN1yy236JFHHlGHDh00ffp0vfTSS3I6nV6pFWhLCDcALuvrLTR1CgsLNX78eO3Zs0eLFy/WW2+9pQ0bNujxxx+XJLnd7st+bmhoaIP7jSY8oeJKzjXDggUL9OWXX2rJkiWKjIzUgw8+qP79+2vXrl2SagZJr1q1Sps3b9a8efN0+vRp3X333Ro+fDhT0YFmItwAaJGMjAydO3dOy5cv1/z58/Wd73xHkyZNqtfNZKbExERFRkbqyJEjF73X0L6WSEtLkyRlZmZe9F5mZqbn/To9e/bUT3/6U61fv1779u1TZWWlnnrqqXrHjBkzRo899pi2b9+uFStWaP/+/Vq5cqVX6gXaCsINgBapazn5ektJZWWl/vSnP5lVUj2hoaGaNGmS1qxZo5ycHM/+I0eOaN26dV75GSNGjFBiYqKee+65et1H69at08GDBzVt2jRJNc8FqqioqHduz549FRsb6znvwoULF7U6DR06VJLomgKaiangAFpk7Nixio+P15w5c/Rf//VfslgseuWVVwKqW2jRokVav369xo0bp/vvv18ul0vPPvusBg0apN27dzfpM6qqqvTb3/72ov3t27fXAw88oMcff1x33XWXxo8fr5kzZ3qmgnfr1k0//vGPJUlffvmlJk6cqO9+97saMGCAwsLCtHr1auXn5+v222+XJP3tb3/Tn/70J910003q2bOniouL9eKLL8pms+mGG27w2u8EaAsINwBaJCEhQW+//bZ++tOf6je/+Y3i4+N1xx13aOLEiZoyZYrZ5UmShg8frnXr1ulnP/uZHnzwQaWmpmrx4sU6ePBgk2ZzSTWtUQ8++OBF+3v27KkHHnhAd955p6KiorR06VL94he/UHR0tG666SY9/vjjnhlQqampmjlzpjZu3KhXXnlFYWFh6tevn15//XXdcsstkmoGFH/++edauXKl8vPzZbfbNWrUKK1YsULdu3f32u8EaAtYWwpAmzNjxgzt379fhw8fNrsUAD7AmBsAQa28vLze68OHD+udd97RhAkTzCkIgM/RcgMgqKWkpOjOO+9Ujx49dPLkSS1btkxOp1O7du1S7969zS4PgA8w5gZAUJs6dar+/ve/Ky8vT1arVenp6fqf//kfgg0QxGi5AQAAQYUxNwAAIKgQbgAAQFBpc2Nu3G63cnJyFBsbK4vFYnY5AACgCQzDUHFxsTp16qSQkMbbZtpcuMnJyVFqaqrZZQAAgBbIzs5Wly5dGj2mzYWb2NhYSTW/HJvNZnI1AACgKRwOh1JTUz1/xxvT5sJNXVeUzWYj3AAA0Mo0ZUgJA4oBAEBQIdwAAICgQrgBAABBhXADAACCCuEGAAAEFcINAAAIKoQbAAAQVAg3AAAgqBBuAABAUCHcAACAoEK4AQAAQYVwAwAAggrhxkuqXG7lOyqUda7M7FIAAGjTCDdesu3EeY3+n426+2/bzC4FAIA2jXDjJXHtIiRJhWVVJlcCAEDbRrjxkriocElSUXmlDMMwuRoAANouwo2X1IWbKpehskqXydUAANB2EW68pF14qCJCa36dheV0TQEAYBbCjZdYLBZP601hWaXJ1QAA0HYRbrzIM+6GQcUAAJiGcONFnhlTdEsBAGAawo0X2T3dUoQbAADMQrjxorh2teGmnDE3AACYhXDjRYy5AQDAfIQbL4qLqhlzc4HZUgAAmCZgws3SpUtlsVi0YMGCSx6zfPlyWSyWeltkZKT/irwMezvG3AAAYLYwswuQpG3btun555/X4MGDL3uszWZTZmam57XFYvFlac3iec4Ns6UAADCN6S03JSUlmjVrll588UXFx8df9niLxaLk5GTPlpSU5Icqm6ZuKjhjbgAAMI/p4Wbu3LmaNm2aJk2a1KTjS0pKlJaWptTUVE2fPl379+9v9Hin0ymHw1Fv85WvWm4YcwMAgFlMDTcrV67Uzp07tWTJkiYd37dvX/31r3/V2rVr9eqrr8rtdmvs2LE6derUJc9ZsmSJ7Ha7Z0tNTfVW+RdhzA0AAOYzLdxkZ2dr/vz5WrFiRZMHBaenp2v27NkaOnSoxo8frzfffFMdO3bU888/f8lzFi5cqKKiIs+WnZ3trUu4SF3LjbParYoqVgYHAMAMpg0o3rFjhwoKCjRs2DDPPpfLpY8++kjPPvusnE6nQkNDG/2M8PBwXX311Tpy5Mglj7FarbJarV6ruzEx1jCFhljkchsqLKtSsr3x+gEAgPeZFm4mTpyovXv31tt31113qV+/fvrFL35x2WAj1YShvXv36oYbbvBVmc1isVgU1y5c50orVVheqWR74ExTBwCgrTAt3MTGxmrQoEH19kVHRyshIcGzf/bs2ercubNnTM7ixYs1ZswY9erVS4WFhXryySd18uRJ3XPPPX6v/1LsUbXhhnE3AACYIiCec3MpWVlZCgn5aljQhQsXdO+99yovL0/x8fEaPny4PvvsMw0YMMDEKuuLY1AxAACmCqhwk5GR0ejr3/3ud/rd737nv4JaoG4JhiKmgwMAYArTn3MTbGi5AQDAXIQbL6truWEJBgAAzEG48TLPU4ppuQEAwBSEGy+rCzeMuQEAwByEGy9jCQYAAMxFuPEyz5gbwg0AAKYg3HhZ3WypIgYUAwBgCsKNl301oJgxNwAAmIFw42Vx7Wq6pUorXaqsdptcDQAAbQ/hxstiI8NksdR8T9cUAAD+R7jxspAQy9dmTNE1BQCAvxFufMCzBAMtNwAA+B3hxgfsTAcHAMA0hBsfiKNbCgAA0xBufOCrJRhouQEAwN8INz4QxxIMAACYhnDjA54xNyyeCQCA3xFufICWGwAAzEO48QHG3AAAYB7CjQ98tb4U4QYAAH8j3PhAHGNuAAAwDeHGBxhzAwCAeQg3PlDXclNcUa1qFyuDAwDgT4QbH7BFhnm+d1RUm1gJAABtD+HGB8JCQxRbG3BYggEAAP8i3PiIZ8YU08EBAPArwo2PxLWrGXdTxKBiAAD8inDjI1+13NAtBQCAPxFufMTOdHAAAExBuPERnlIMAIA5CDc+4hlzw4BiAAD8inDjI1+13DDmBgAAfyLc+EjdmJsLdEsBAOBXhBsf+WrxTMINAAD+RLjxkbpuqSK6pQAA8CvCjY94Vgan5QYAAL8i3PiIva7lprxKbrdhcjUAALQdhBsfqRtQbBhSMSuDAwDgN4QbH7GGhSoqIlQSSzAAAOBPhBsfimMJBgAA/C5gws3SpUtlsVi0YMGCRo9744031K9fP0VGRuqqq67SO++8458CW4Dp4AAA+F9AhJtt27bp+eef1+DBgxs97rPPPtPMmTP1gx/8QLt27dKMGTM0Y8YM7du3z0+VNg9PKQYAwP9MDzclJSWaNWuWXnzxRcXHxzd67NNPP62pU6fq5z//ufr3769HH31Uw4YN07PPPuunapsn7mszpgAAgH+YHm7mzp2radOmadKkSZc9dvPmzRcdN2XKFG3evPmS5zidTjkcjnqbv9hrF89kzA0AAP4TZuYPX7lypXbu3Klt27Y16fi8vDwlJSXV25eUlKS8vLxLnrNkyRI98sgjV1RnS33VLUW4AQDAX0xrucnOztb8+fO1YsUKRUZG+uznLFy4UEVFRZ4tOzvbZz/rm756SjFjbgAA8BfTWm527NihgoICDRs2zLPP5XLpo48+0rPPPiun06nQ0NB65yQnJys/P7/evvz8fCUnJ1/y51itVlmtVu8W30RfrS9Fyw0AAP5iWsvNxIkTtXfvXu3evduzjRgxQrNmzdLu3bsvCjaSlJ6ero0bN9bbt2HDBqWnp/ur7GbxjLlhQDEAAH5jWstNbGysBg0aVG9fdHS0EhISPPtnz56tzp07a8mSJZKk+fPna/z48Xrqqac0bdo0rVy5Utu3b9cLL7zg9/qbgqngAAD4n+mzpRqTlZWl3Nxcz+uxY8fqtdde0wsvvKAhQ4Zo1apVWrNmzUUhKVAwFRwAAP+zGIbRppasdjgcstvtKioqks1m8+nPyiuq0JglGxUWYtHhx66XxWLx6c8DACBYNefvd0C33LR2dS031W5DpZUuk6sBAKBtINz4UGR4qKxhNb9ixt0AAOAfhBsf40F+AAD4F+HGx+JYggEAAL8i3PiYPYqnFAMA4E+EGx/zLMFAyw0AAH5BuPExnnUDAIB/EW58LC6qbswN3VIAAPgD4cbHmC0FAIB/EW58LI7FMwEA8CvCjY95xtzQcgMAgF8QbnzMM1uKqeAAAPgF4cbH7Iy5AQDArwg3PuaZLVVepTa2ADsAAKYg3PhYXbdUZbVbFVVuk6sBACD4EW58LCoiVOGhFkmMuwEAwB8INz5msVhkZ/FMAAD8hnDjBzzIDwAA/yHc+EHduJsiuqUAAPA5wo0f0HIDAID/EG78wM4SDAAA+A3hxg9ouQEAwH8IN37AmBsAAPyHcOMHtNwAAOA/hBs/sNcuwXChjJYbAAB8jXDjB56VwWm5AQDA5wg3flDXLVXEbCkAAHyOcOMH8VEsvwAAgL8QbvzAXttyU17lUkWVy+RqAAAIboQbP4i1hik0pGZlcAddUwAA+BThxg9qVgavHVRMuAEAwKcIN37CjCkAAPyDcOMnds+D/HjWDQAAvkS48ZM4uqUAAPALwo2fxNVOBy+iWwoAAJ8i3PjJVwOK6ZYCAMCXCDd+wuKZAAD4B+HGTxhzAwCAfxBu/IQxNwAA+Afhxk88U8EZcwMAgE+ZGm6WLVumwYMHy2azyWazKT09XevWrbvk8cuXL5fFYqm3RUZG+rHiluMhfgAA+EeYmT+8S5cuWrp0qXr37i3DMPS3v/1N06dP165duzRw4MAGz7HZbMrMzPS8tlgs/ir3itAtBQCAf5gabm688cZ6rx977DEtW7ZMW7ZsuWS4sVgsSk5O9kd5XlXXclPsrFaVy63wUHoEAQDwhYD5C+tyubRy5UqVlpYqPT39kseVlJQoLS1Nqampmj59uvbv39/o5zqdTjkcjnqbGWy14UZiZXAAAHzJ9HCzd+9excTEyGq16r777tPq1as1YMCABo/t27ev/vrXv2rt2rV69dVX5Xa7NXbsWJ06deqSn79kyRLZ7XbPlpqa6qtLaVRoiEW2yJqGMqaDAwDgOxbDMAwzC6isrFRWVpaKioq0atUq/fnPf9amTZsuGXC+rqqqSv3799fMmTP16KOPNniM0+mU0+n0vHY4HEpNTVVRUZFsNpvXrqMprn3iQ2WdL9M/7x+r4Wnxfv3ZAAC0Zg6HQ3a7vUl/v00dcyNJERER6tWrlyRp+PDh2rZtm55++mk9//zzlz03PDxcV199tY4cOXLJY6xWq6xWq9fqvRJxUeHKOs/K4AAA+JLp3VLf5Ha767W0NMblcmnv3r1KSUnxcVXeUTdjiungAAD4jqktNwsXLtT111+vrl27qri4WK+99poyMjL03nvvSZJmz56tzp07a8mSJZKkxYsXa8yYMerVq5cKCwv15JNP6uTJk7rnnnvMvIwmYwkGAAB8z9RwU1BQoNmzZys3N1d2u12DBw/We++9p+uuu06SlJWVpZCQrxqXLly4oHvvvVd5eXmKj4/X8OHD9dlnnzVpfE4gqFs8s4huKQAAfMb0AcX+1pwBSd72f+sz9cwHRzQ7PU2Lpw/y688GAKA1a87f74AbcxPM7Iy5AQDA5wg3fsSYGwAAfI9w40eMuQEAwPcIN35UF25ouQEAwHcIN35kb8eYGwAAfI1w40d1LTeOiiq53G1qkhoAAH5DuPEje+2AYsOQiitovQEAwBcIN34UHhqiGGvtyuB0TQEA4BOEGz+zMx0cAACfItz4mWfGFNPBAQDwCcKNn3medUPLDQAAPkG48bM4poMDAOBThBs/s3u6pQg3AAD4AuHGz75aX4oxNwAA+ALhxs++Wl+KlhsAAHyBcONncVG1Y24YUAwAgE8QbvzM0y3FVHAAAHyCcONnnpYbuqUAAPAJwo2feR7iR7cUAAA+Qbjxs693S7lZGRwAAK8j3PiZrTbcuA2ppLLa5GoAAAg+hBs/iwwPVbvwUElMBwcAwBcINyaI4ynFAAD4DOHGBHaeUgwAgM8QbkxAyw0AAL5DuDGBZ2VwpoMDAOB1hBsTfLW+FN1SAAB4G+HGBHa6pQAA8BnCjQnolgIAwHcINyZgQDEAAL5DuDFB3RIMRUwFBwDA6wg3JmDMDQAAvtOicJOdna1Tp055Xn/++edasGCBXnjhBa8VFswYcwMAgO+0KNx8//vf14cffihJysvL03XXXafPP/9cv/71r7V48WKvFhiMvpoKXiXDYGVwAAC8qUXhZt++fRo1apQk6fXXX9egQYP02WefacWKFVq+fLk36wtK8VE1LTeVLrfKq1wmVwMAQHBpUbipqqqS1WqVJL3//vv6j//4D0lSv379lJub673qglRkeIgiwmp+9edKGFQMAIA3tSjcDBw4UM8995w+/vhjbdiwQVOnTpUk5eTkKCEhwasFBiOLxaKu7aMkSUfOlJhcDQAAwaVF4ebxxx/X888/rwkTJmjmzJkaMmSIJOlf//qXp7sKjRvYySZJ2n+6yORKAAAILmEtOWnChAk6e/asHA6H4uPjPft/+MMfKioqymvFBbOBnWxauztH+3McZpcCAEBQaVHLTXl5uZxOpyfYnDx5Ur///e+VmZmpxMTEJn/OsmXLNHjwYNlsNtlsNqWnp2vdunWNnvPGG2+oX79+ioyM1FVXXaV33nmnJZdgukGd7JJEuAEAwMtaFG6mT5+ul19+WZJUWFio0aNH66mnntKMGTO0bNmyJn9Oly5dtHTpUu3YsUPbt2/Xt7/9bU2fPl379+9v8PjPPvtMM2fO1A9+8APt2rVLM2bM0IwZM7Rv376WXIapBtR2S2WdL5OjgufdAADgLS0KNzt37tQ111wjSVq1apWSkpJ08uRJvfzyy3rmmWea/Dk33nijbrjhBvXu3Vt9+vTRY489ppiYGG3ZsqXB459++mlNnTpVP//5z9W/f389+uijGjZsmJ599tmWXIap4qIi1DmunSTpAK03AAB4TYvCTVlZmWJjYyVJ69ev180336yQkBCNGTNGJ0+ebFEhLpdLK1euVGlpqdLT0xs8ZvPmzZo0aVK9fVOmTNHmzZtb9DPN5hlUTLgBAMBrWhRuevXqpTVr1ig7O1vvvfeeJk+eLEkqKCiQzWZr1mft3btXMTExslqtuu+++7R69WoNGDCgwWPz8vKUlJRUb19SUpLy8vIu+flOp1MOh6PeFigGesbdMGMKAABvaVG4eeihh/Szn/1M3bp106hRozwtLevXr9fVV1/drM/q27evdu/era1bt+r+++/XnDlzdODAgZaU1aAlS5bIbrd7ttTUVK999pWqa7mhWwoAAO9pUbi59dZblZWVpe3bt+u9997z7J84caJ+97vfNeuzIiIi1KtXLw0fPlxLlizRkCFD9PTTTzd4bHJysvLz8+vty8/PV3Jy8iU/f+HChSoqKvJs2dnZzarPlwZ2rgk3hwtKVMEyDAAAeEWLwo1UEzSuvvpq5eTkeFYIHzVqlPr163dFBbndbjmdzgbfS09P18aNG+vt27BhwyXH6EiS1Wr1TDWv2wJFsi1S7aMj5HIbyswrNrscAACCQovCjdvt1uLFi2W325WWlqa0tDTFxcXp0UcfldvtbvLnLFy4UB999JFOnDihvXv3auHChcrIyNCsWbMkSbNnz9bChQs9x8+fP1/vvvuunnrqKR06dEiLFi3S9u3bNW/evJZchuksFguDigEA8LIWPaH417/+tf7yl79o6dKlGjdunCTpk08+0aJFi1RRUaHHHnusSZ9TUFCg2bNnKzc3V3a7XYMHD9Z7772n6667TpKUlZWlkJCv8tfYsWP12muv6Te/+Y1+9atfqXfv3lqzZo0GDRrUkssICAM72fXx4bMMKgYAwEsshmEYzT2pU6dOeu655zyrgddZu3atHnjgAZ0+fdprBXqbw+GQ3W5XUVFRQHRRvbUnR//5910amhqnNXPHmV0OAAABqTl/v1vULXX+/PkGx9b069dP58+fb8lHtll13VKH8hxyuZudMwEAwDe0KNwMGTKkwacCP/vssxo8ePAVF9WWdEuIVnREqCqq3Dp2psTscgAAaPVaNObmiSee0LRp0/T+++97Zipt3rxZ2dnZrXYhS7OEhFjUP8Wm7ScvaH+OQ72TYs0uCQCAVq1FLTfjx4/Xl19+qZtuukmFhYUqLCzUzTffrP379+uVV17xdo1Br65rat9pBhUDAHClWtRyI9UMKv7mrKg9e/boL3/5i1544YUrLqwt+WoZBqaDAwBwpVr8ED94zwDPs26K1ILJawAA4GsINwGgT1KswkMtclRU69SFcrPLAQCgVSPcBICIsBD1qR1ITNcUAABXplljbm6++eZG3y8sLLySWtq0gZ1s2p/j0IGcIk0ddOmFQAEAQOOaFW7sdvtl3589e/YVFdRW1QwqPkXLDQAAV6hZ4eall17yVR1tHgtoAgDgHYy5CRD9U2yyWKQ8R4XOljjNLgcAgFaLcBMgoq1h6p4QLYnWGwAArgThJoB8/Xk3AACgZQg3AYQnFQMAcOUINwGkblDxAcINAAAtRrgJIHXh5vjZUpU4q02uBgCA1olwE0ASYqxKsUdKkg7m0noDAEBLEG4CjOd5N6cZVAwAQEsQbgLMAAYVAwBwRQg3Aaau5WYf4QYAgBYh3ASYunBzOL9YzmqXydUAAND6EG4CTOe4drK3C1e129Dh/BKzywEAoNUh3AQYi8XytUU0GVQMAEBzEW4CECuEAwDQcoSbADSoMzOmAABoKcJNAKpruTmY65DLbZhcDQAArQvhJgB17xCjduGhKqt06cS5UrPLAQCgVSHcBKDQEIv6pcRKkvbxpGIAAJqFcBOgWCEcAICWIdwEqIEswwAAQIsQbgLU1591YxgMKgYAoKkINwGqT1KsQkMsulBWpdyiCrPLAQCg1SDcBKjI8FD1ToyRRNcUAADNQbgJYANYhgEAgGYj3ASwQQwqBgCg2Qg3AcwzqJhn3QAA0GSEmwBW1y2VU1ShC6WVJlcDAEDrQLgJYLGR4UpLiJJE1xQAAE1FuAlwAxlUDABAs5gabpYsWaKRI0cqNjZWiYmJmjFjhjIzMxs9Z/ny5bJYLPW2yMhIP1XsfzypGACA5jE13GzatElz587Vli1btGHDBlVVVWny5MkqLW18JWybzabc3FzPdvLkST9V7H9MBwcAoHnCzPzh7777br3Xy5cvV2Jionbs2KFrr732kudZLBYlJyf7uryAUNctdexsqcoqqxUVYeotAwAg4AXUmJuioprWifbt2zd6XElJidLS0pSamqrp06dr//79lzzW6XTK4XDU21qTxNhIdYy1yjCkg7nFZpcDAEDAC5hw43a7tWDBAo0bN06DBg265HF9+/bVX//6V61du1avvvqq3G63xo4dq1OnTjV4/JIlS2S32z1bamqqry7BZwbVtt7syrpgciUAAAQ+ixEgS07ff//9WrdunT755BN16dKlyedVVVWpf//+mjlzph599NGL3nc6nXI6nZ7XDodDqampKioqks1m80rtvvbSp8f1yFsHNKp7e73+o3SzywEAwO8cDofsdnuT/n4HRMvNvHnz9Pbbb+vDDz9sVrCRpPDwcF199dU6cuRIg+9brVbZbLZ6W2tz3YAkSdL2E+d1rsR5maMBAGjbTA03hmFo3rx5Wr16tT744AN179692Z/hcrm0d+9epaSk+KDCwNAlPkqDOtvkNqSNBwvMLgcAgIBmariZO3euXn31Vb322muKjY1VXl6e8vLyVF5e7jlm9uzZWrhwoef14sWLtX79eh07dkw7d+7UHXfcoZMnT+qee+4x4xL8ZvKAmtlh6w/kmVwJAACBzdRws2zZMhUVFWnChAlKSUnxbP/4xz88x2RlZSk3N9fz+sKFC7r33nvVv39/3XDDDXI4HPrss880YMAAMy7BbyYPrOma+ujwWZU6q02uBgCAwBUwA4r9pTkDkgKJYRia8L8ZOnmuTMtmDdP1VwVvNxwAAN/U6gYU4/IsFosm1w4sXn8g3+RqAAAIXISbVmTKwJpxNxsP5qvK5Ta5GgAAAhPhphW5umu8OsREyFFRra3HzptdDgAAAYlw04qEhlg0qX9d1xSzpgAAaAjhppWp65pavz9fbnebGgsOAECTEG5amfSeCYqOCFWeo0J7TxeZXQ4AAAGHcNPKRIaHakLfREl0TQEA0BDCTStU90C/9/YzJRwAgG8i3LRC3+qXqPBQi44UlOjomRKzywEAIKAQblohW2S40nt2kCRt4IF+AADUQ7hppeqeVvzefsbdAADwdYSbVuq62nCzK6tQBY4Kk6sBACBwEG5aqSRbpK7uGieJtaYAAPg6wk0rNnlA7QP9CDcAAHgQblqxKbVTwjcfPStHRZXJ1QAAEBgIN61Yj44x6pUYoyqXoQ8PFZhdDgAAAYFw08rVzZqiawoAgBqEm1aubiHNjEMFqqhymVwNAADmI9y0cld1tivZFqnSSpc2Hz1ndjkAAJiOcNPKhYRYPM+8YSFNAAAIN0Ghrmtqw4F8udyGydUAAGAuwk0QGN2jvWyRYTpbUqldWRfMLgcAAFMRboJAeGiIJvZn1hQAABLhJmh8fSFNw6BrCgDQdhFugsS1fToqIixEJ8+V6cv8ErPLAQDANISbIBFtDdO1vTtIqmm9AQCgrSLcBJGvFtIk3AAA2i7CTRCZ2D9RIRZp32mHTheWm10OAACmINwEkYQYq0Z0ay9JWk/XFACgjSLcBJm6B/q9suWkqlxuk6sBAMD/CDdB5rYRXZQQHaFjZ0q1YstJs8sBAMDvCDdBxhYZrh9f10eS9PuNh1VUVmVyRQAA+BfhJgjdPjJVfZJiVFhWpWc+OGx2OQAA+BXhJgiFhYboN9MGSJJe3nxCx8+WmlwRAAD+Q7gJUtf26ahv9e2oKpeh/3nnoNnlAADgN4SbIPbraf0VGmLRhgP5+uzoWbPLAQDALwg3QaxXYqzuGN1VkvTo2wflcrOgJgAg+BFugtyCSX1kiwzTwVyHVu3INrscAAB8jnAT5OKjI/RfE3tLkp5870uVOKtNrggAAN8i3LQBs9O7qVtClM6WOLUs44jZ5QAA4FOmhpslS5Zo5MiRio2NVWJiombMmKHMzMzLnvfGG2+oX79+ioyM1FVXXaV33nnHD9W2XhFhIVp4Q39J0osfH9epC2UmVwQAgO+YGm42bdqkuXPnasuWLdqwYYOqqqo0efJklZZe+rksn332mWbOnKkf/OAH2rVrl2bMmKEZM2Zo3759fqy89Zk8IEljerRXZbVbj797+QAJAEBrZTEMI2Cm0Jw5c0aJiYnatGmTrr322gaP+d73vqfS0lK9/fbbnn1jxozR0KFD9dxzz132ZzgcDtntdhUVFclms3mt9tZgf06RvvOHT2QY0j/vH6vhafFmlwQAQJM05+93QI25KSoqkiS1b9/+ksds3rxZkyZNqrdvypQp2rx5c4PHO51OORyOeltbNbCTXbcN7yJJevTtA3IzNRwAEIQCJty43W4tWLBA48aN06BBgy55XF5enpKSkurtS0pKUl5eXoPHL1myRHa73bOlpqZ6te7W5meT+yoqIlS7swv11hc5ZpcDAIDXBUy4mTt3rvbt26eVK1d69XMXLlyooqIiz5ad3baf9ZJoi9QDE3pKkh5fd0jllS6TKwIAwLsCItzMmzdPb7/9tj788EN16dKl0WOTk5OVn59fb19+fr6Sk5MbPN5qtcpms9Xb2rp7rumhznHtlFNUob98cszscgAA8CpTw41hGJo3b55Wr16tDz74QN27d7/sOenp6dq4cWO9fRs2bFB6erqvygw6keGh+u+pfSVJf8o4qgJHhckVAQDgPaaGm7lz5+rVV1/Va6+9ptjYWOXl5SkvL0/l5eWeY2bPnq2FCxd6Xs+fP1/vvvuunnrqKR06dEiLFi3S9u3bNW/ePDMuodX6jyGddHXXOJVVurT03UNmlwMAgNeYGm6WLVumoqIiTZgwQSkpKZ7tH//4h+eYrKws5ebmel6PHTtWr732ml544QUNGTJEq1at0po1axodhIyLWSwWPfidAZKkN3ee1jMbD5tcEQAA3hFQz7nxh7b8nJuG/PnjY/rtvw9Kkn55fT/dN76nyRUBAHCxVvucG/jfPdf00M+n1Iy/WbrukP7yyXGTKwIA4MoQbqC53+ql+bUrhz/69gG9svmEuQUBAHAFCDeQJC2Y1Fv31z7/5sG1+7Xy8yyTKwIAoGUIN5BUM8D4v6f01T3/r2Y6/sLVe/XPHadMrgoAgOYj3MDDYrHo19P6a056mgxD+vmqPfrXHpZoAAC0LoQb1GOxWPTwjQM1c1Sq3Ib043/s1rq9uZc/EQCAAEG4wUVCQix6bMZVumVYF7nchv7z77v0/oH8y58IAEAAINygQSEhFj1x62BNH9pJ1W5DD6zYqYzMArPLAgDgsgg3uKTQEIueum2IbrgqWZUut374yg59euSs2WUBANAowg0aFRYaoqdvv1qT+iepstqtu5dv07KMo6pyuc0uDQCABhFucFnhoSH646yagOOsduvxdw/pO898ou0nzptdGgAAFyHcoEmsYaF6cfZwPXnrYMVHhSszv1i3PrdZv/znFyosqzS7PAAAPAg3aDKLxaLbRqRq408n6LsjukiSVm7L1sSnNunNnafUxtZgBQAEKMINmq19dISeuHWI/vHDMeqVGKNzpZX6yet79P0Xt+romRKzywMAtHGEG7TY6B4Jeue/rtHPp/SVNSxEm4+d0/W//1j/t+FLVVS5zC4PANBGEW5wRSLCQjT3W7204cfjNaFvR1W63Hpm42FN/f1H+uQw08YBAP5HuIFXdE2I0kt3jtQfvz9MibFWnThXpjv+slX3v7pDWefKzC4PANCGWIw2NgrU4XDIbrerqKhINpvN7HKCUnFFlZ5a/6Ve3nxCbkOKCA3RXeO6ae63e8kWGW52eQCAVqg5f78JN/CZQ3kOPfbvg/q4tnsqITpCP76uj24fmaqwUBoNAQBNR7hpBOHGvwzD0IeZBfrtvw/q2JlSSVKfpBj9ZtoAXduno8nVAQBaC8JNIwg35qhyubViy0n9fuNhFZZVSZK+1bejfj1tgHolxphcHQAg0BFuGkG4MVdhWaWe2XhEL28+oWq3odAQi+4Y3VULJvVRfHSE2eUBAAIU4aYRhJvAcOxMif7nnUN6/2C+JMkWGaY7x3XXbcO7KLV9lMnVAQACDeGmEYSbwPLpkbN69O0DOpRX7Nk3tmeCvjsiVVMGJqtdRKiJ1QEAAgXhphGEm8Djchv6995c/WNblj49cs6zP9YaphuHdtJtw7toaGqcLBaLiVUCAMxEuGkE4SawZZ8v0z93ntIb20/pdGG5Z3/vxBjdNqKLbrq6izrGWk2sEABgBsJNIwg3rYPbbWjLsXN6fXu21u3Lk7PaLUkKDbHoW30T9R9DO2lM9/ZKtEWaXCkAwB8IN40g3LQ+jooqvbUnR29sP6Xd2YX13ktLiNLIbu01qlt7jezeXt0Soui+AoAgRLhpBOGmdTucX6xVO07p48NndTDPoW/+r7djrLUm6HSL18ju7dUv2abQEMIOALR2hJtGEG6Ch6OiSjtOXtDnx89r2/Hz+uJUkSpd7nrHxFrDNLpHe906vIsm9k9SOMs+AECrRLhpBOEmeFVUubQnu1DbTpzX5ycuaOfJCypxVnveT7JZ9b2RXTVzVKpS7O1MrBQA0FyEm0YQbtqOapdbh/KK9e+9uXp9W7bOlVZKqhmUPLFfomaNSdM1vToohG4rAAh4hJtGEG7aJme1S+/tz9eKLSe19fh5z/6u7aP0/dFdddvwLkqIYYo5AAQqwk0jCDc4nF+sFVuz9M8dp1Rc220VERqiG65K1qwxaRqRFs+MKwAIMISbRhBuUKesslpv7cnRq1uytPd0kWd/v+RYzRnbTdOHdlJURJiJFQIA6hBuGkG4QUO+OFWoV7ec1L/25KiiqmbGlS0yTN8dkao7xqSpW4dokysEgLaNcNMIwg0aU1RWpTd2ZOvlzSeVdb5MkmSxSBP6dNTssd00vnfHZg1Arqx26+iZEh3Mdaiy2q0BnWzqkxSryHAWBAWA5iDcNIJwg6Zwuw1t+vKM/rb5hDIyz3j2pyVE6f8bk6bbhqfKHhVe75wLpZU6mOvQgdrtYG6xjhQUq8pV/z+xsBCL+iTFalBnmwZ1tmtQZ7v6J9tYAR0AGkG4aQThBs11/GypXt1yUq9vz1ZxRc0A5MjwEM0Y2lkJMRE6kFMTZPIcFQ2eHxsZpv4pNlnDQrTvdJEulFVddEyIReqVGKNBnWrCTt/kWEVbw2QNC1FEWMjXvobWfB8awhR2AG0K4aYRhBu0VFlltdbsytHLm0/oUF5xg8ekJUSpf7JN/VNs6p8SqwGdbOoc184z+8owDOUUVWjvqSLtzynSvtNF2nvaobMlzmbXExFaE3jaRYRq8oAk/dfE3kpiIVEAQarVhJuPPvpITz75pHbs2KHc3FytXr1aM2bMuOTxGRkZ+ta3vnXR/tzcXCUnJzfpZxJucKUMw9Dnx8/rzZ2nFRIiDUipCTN9k2MVGxl++Q9o4PMKip21QadI+047dOxsiZxVbjmr3XJWu1RZ7fasjH4pkeEhmjO2m+4f31NxUREtvTwACEjN+ftt6jzX0tJSDRkyRHfffbduvvnmJp+XmZlZ78ISExN9UR7QIIvFotE9EjS6R4LXPi/JFqkkW6Qm9k+65HGGYajKZdQLO5XVbmWdL9MzGw9r+8kLen7TMb22NUv3je+pu8Z1u6Kp7G63oSq3W9YwxgIBaF1MDTfXX3+9rr/++mafl5iYqLi4OO8XBAQwi8WiiDCLIsLqL/7ZrUO0rundQR8cKtCT72XqUF6xnnwvUy99ekL/NbGXbh/Z9aJzLiXfUaGPvjyjjw6f1SeHz6iovEoDO9k1olu8RnVrrxHd2qtjLE9yBhDYWuUTyoYOHSqn06lBgwZp0aJFGjdu3CWPdTqdcjq/Gs/gcDj8USLgVxaLRRP7J+lbfRP1rz05+r8NXyrrfJkeWrtfL358TD+e1EfTh3ZW6DcGITurXdp2/II+OnxGH315psGxRHtru8te+vSEJKlHh2iN7Na+JvB0b6+u7aN4ojOAgBIwA4otFstlx9xkZmYqIyNDI0aMkNPp1J///Ge98sor2rp1q4YNG9bgOYsWLdIjjzxy0X7G3CCYVVa79Y/t2Xpm42GdKa4J932TYvWzKX3Vo2N0TevMl2e05dh5lVe5POdZLNLgznZd26ejru3TUSn2SO3MKtS24+e17cR5ZeYX65v/YiTGWjWye3uNTIvXwNqZXrYWjD0CgMa0mgHFX9eUcNOQ8ePHq2vXrnrllVcafL+hlpvU1FTCDdqEsspqLf/shJ7LOCpH7TT2b0qMteqa3h11bZ8OuqZ3R7WPvvRg5KKyKm0/eV7bTlzQthPn9cWpwoue4yNJnePaqV9yrPqlxKpvsk39k2PVvUO0wkKb1j0GAN/UagYUe8OoUaP0ySefXPJ9q9Uqq5UxAmiboiLC9MCEXpo1Kk3PfXRUL316XG63NLJ7vK7tXdM60y85tsndSvaocE3sn+QZ+FxR5dLu7JqWnV3ZhTqU61BOUYVOF5brdGG5Nh4q8JwbERaiXh1jagJPUqwiwkJUVulSqbO6Zqt0qayyWqXO+l9LnC653G6FhlhqNotFISEWhYXUfA21WDzvhYVYFBMZpmFd4zW6e4KGpcWxPhjQBrX6lpvrrrtOsbGxevPNN5t0PFPB0ZY5q10yDPl0+Yeisipl5hfrUF7Nww0z8xzKzCtWaaXr8id7WViIRYO72Gtmt3WvGRAdYyXsAK1Rq2m5KSkp0ZEjRzyvjx8/rt27d6t9+/bq2rWrFi5cqNOnT+vll1+WJP3+979X9+7dNXDgQFVUVOjPf/6zPvjgA61fv96sSwBaFX9M67ZHhWtU9/Ya1b29Z5/bbejUhXIdynPoUF6xDheUyDAMRUeEKcoaquiIMEVbwxRtDVVURJiiI0IVZa39GhGmsFCLXG5DLrcht2Go2m3IXfva5TbkMr76vqDYqc+Pn9fWY+eUU1ShnVmF2plVqGUZRxUaYtGgTjaN6t5eo7snaGT39rK3Y3wQEGxMDTfbt2+v91C+n/zkJ5KkOXPmaPny5crNzVVWVpbn/crKSv30pz/V6dOnFRUVpcGDB+v9999v8MF+AAJHSIhFXROi1DUhSpMHNu2Bm1di5qiuMoyaQLXl2DltPX5eW4+fU/b5cu05VaQ9p4r04sfHZbFI3RKiNSDFpgGdap8qnWJXks3a5K46wzB0urBch3KLlZlfrIO5NQHubIlThlHzviFJhmToq9eGIRkyZBjyPGX6thGpGpEWz+wz4AoFTLeUv9AtBbRdOYXl2nr8nLYeO6+tx8/r+NnSBo9rHx1RG3TqQo9NPTvGqKLKpS/zi3Uwt6bbLTOvWIdyi1XsbHiwdkt0S4jSrcO76KZhXdQ5rp3XPhdo7VrlbCl/IdwAqHO2xFmzkntO7WruOQ4dO1sql/vifxbDQy0Nzgyre69nx5jaGWI29UuO9awpZrFIFtWMK6z5KllUs7/OqQvlenPnKf17b67KascmWSzSuJ4ddNuILpo8IJlV4/2g2uVW9oVyHSko0fGzJbLIovbREWofE6GE6AjFR0UoISaCQeomIdw0gnADoDF1rTM1q73XhJ6DucUqqW2dSbZF1k5xj1X/ZJv6pcSqR4eYJj8FujGlzmq9uy9Pb+zI1pZj5z37Y61h+s6QFN06PFXDusZd1G1lGIac1W6V1M48K66o+VrirFZoiEWxkeGyRYYpNjJcsZFhiooI9WnX1/nSSu3JLtSu7ELtzi7UgRyHOsVFamzPDhrXK0Eju7X36aD2yymrrNbRglIdPVOiIwUlOnqmZjtxtkyVrsbXcJNq1nFLiLaqfXSE4qNrgk/nuHYa3MWuoV3jlBjLAra+QLhpBOEGQHO53TXjamKsYYpv5DlA3pR9vkz/3HlKq3ac0qkL5Z793RKilBBjVUlFTXipCzTVDbQ2XUpoiEUx1jDFfi3w2CLDZG8XoUSbVUmxViXaIpUYa1WSLVIdY62XDCPOapcO5Di0uzbI7M4u1MlzZY3+/IjQEA1Li9O4nh00tlcHDelib9EzkCqr3bpQVqniimoVV1TV/D4qasJdsef7mv3FzmoVllXq+JlS5RRVXPIzI8ND1KNDjHomxijEUhPUzpVU6kJZpc6VVqryMgvYSjXPeRqaGqeru8ZpaGqcBnW2mxrmggXhphGEGwCtidttaOvx81q145Te2Ztb74nSDYmOCFW0NUwx1poZaC63oWJnVW0AqG6wy60pbJFhSrJFKtFmVWJspNpFhGp/jkMHcxwNtnb06Bhd8wc+NU4DOtl18lypPj1yTp8dPavcb4SLGGuYRndvr7G9alp2OsW105lipwocThUUV+hMsdOzFXi+VuhCWVWLrkWSEqIj1DMxRj07xqhnx2j1qv2+c1w7hYQ03KplGIZKK106X1Kp82WVOl/q1LmSSp0vrdTRMyXanV1YOxOw/nlhIRb1S4nV0NQ4DU2NV9+kWDmrXXJUVMlRXl37tUqOiurar1/tL3FWq1tCtK5OjdPQrnEakhoXsE8Ad7sNbT52Tq9tzVK/5Fj958TeXv18wk0jCDcAWqsSZ7U+PXJWhlETCGIiwxRj/VqYiQi75B9mqeaPc3mVy9PSUVRe87Uu+Fwoq9SZYqfyHRUqqA0Q+Q7nZVsr4qPCPX+4r+4apyFd4mSPavgPsGEYOn62VJ8eOatPj5zT5mPnVFTe8pASYpFiI8O/1hIVVvu7qWmRirWGeX5XtshwpSVEqWfHGJ+1wBVXVGnvqSJPl9zu7ELPEijeYLFIPTvG1GsZ6psUa+rTv8+XVmrVjmz9/fNszyD9JJtVn/1y4kXr2V0Jwk0jCDcA0HSGYchRXu0JOgXFNcGnuKJKfZJqWiOuZPFUl9vQgRyHPj16Vp8eOattJ86rosqtWGuYOtqsSoy1qmNsZO1X69e+1nSXxbULbzTQmc0wDOUUVWh3VqF2Z1/Q7uxCHT9bphhrqGztwmWLDJetXVjt15qxUV/fbw0LVWZesScoZZ2/uMuvXXiorupi19WpcUptH6V24aGKDA9VZHjIN77WbmE1r9uFh7b4d2cYhraduKAVW09q3d48T+tdjDVMN13dWd8f3VX9U7z7N5Zw0wjCDQAEriqXW9Uug9lhl3C2xFkzWDurJuzsyS5s8aMIIsJC1KNDtPomx6pPUqx6J8aoT1KsUttHXbLFpaisSv/ceUqvfZ6lIwUlnv2Du9j1/VFddeOQTor20VPACTeNINwAAIKF223o6JkS7aoNOmeKnaqodquiyiVnlUsVVW6VV7lUUbdVuy/bzWgNC1Gv2qBTs8WoXUSo/rnjtN7+IkfO2vOjIkI1fWgnfX9Umq7qYvf5tRJuGkG4AQC0ZW63oYpql84WV+rL/GJ9WVCsw/kl+jK/WEcKSjzh5VL6Jcdq1pg0zRjaSbF+HNzcataWAgAA/hUSYlFURJi6JoSpa0KUJg1I8rznchvKOl+mL/OLdTi/WF/Whp4zxU5N6JuoWWO66urUi5+1FGgINwAAQFLNM5C6d4hW9w7RmuKHdeB8xby5YwAAAD5AuAEAAEGFcAMAAIIK4QYAAAQVwg0AAAgqhBsAABBUCDcAACCoEG4AAEBQIdwAAICgQrgBAABBhXADAACCCuEGAAAEFcINAAAIKoQbAAAQVMLMLsDfDMOQJDkcDpMrAQAATVX3d7vu73hj2ly4KS4uliSlpqaaXAkAAGiu4uJi2e32Ro+xGE2JQEHE7XYrJydHsbGxslgsXv1sh8Oh1NRUZWdny2azefWzA0GwX58U/NfI9bV+wX6NXF/r56trNAxDxcXF6tSpk0JCGh9V0+ZabkJCQtSlSxef/gybzRa0/6OVgv/6pOC/Rq6v9Qv2a+T6Wj9fXOPlWmzqMKAYAAAEFcINAAAIKoQbL7JarXr44YdltVrNLsUngv36pOC/Rq6v9Qv2a+T6Wr9AuMY2N6AYAAAEN1puAABAUCHcAACAoEK4AQAAQYVwAwAAggrhxkv++Mc/qlu3boqMjNTo0aP1+eefm12S1yxatEgWi6Xe1q9fP7PLarGPPvpIN954ozp16iSLxaI1a9bUe98wDD300ENKSUlRu3btNGnSJB0+fNicYlvoctd45513XnRPp06dak6xLbBkyRKNHDlSsbGxSkxM1IwZM5SZmVnvmIqKCs2dO1cJCQmKiYnRLbfcovz8fJMqbp6mXN+ECRMuuof33XefSRU3z7JlyzR48GDPQ97S09O1bt06z/ut+d7Vudw1tub715ClS5fKYrFowYIFnn1m3kfCjRf84x//0E9+8hM9/PDD2rlzp4YMGaIpU6aooKDA7NK8ZuDAgcrNzfVsn3zyidkltVhpaamGDBmiP/7xjw2+/8QTT+iZZ57Rc889p61btyo6OlpTpkxRRUWFnyttuctdoyRNnTq13j39+9//7scKr8ymTZs0d+5cbdmyRRs2bFBVVZUmT56s0tJSzzE//vGP9dZbb+mNN97Qpk2blJOTo5tvvtnEqpuuKdcnSffee2+9e/jEE0+YVHHzdOnSRUuXLtWOHTu0fft2ffvb39b06dO1f/9+Sa373tW53DVKrff+fdO2bdv0/PPPa/DgwfX2m3ofDVyxUaNGGXPnzvW8drlcRqdOnYwlS5aYWJX3PPzww8aQIUPMLsMnJBmrV6/2vHa73UZycrLx5JNPevYVFhYaVqvV+Pvf/25ChVfum9doGIYxZ84cY/r06abU4wsFBQWGJGPTpk2GYdTcs/DwcOONN97wHHPw4EFDkrF582azymyxb16fYRjG+PHjjfnz55tXlJfFx8cbf/7zn4Pu3n1d3TUaRvDcv+LiYqN3797Ghg0b6l2T2feRlpsrVFlZqR07dmjSpEmefSEhIZo0aZI2b95sYmXedfjwYXXq1Ek9evTQrFmzlJWVZXZJPnH8+HHl5eXVu592u12jR48OqvspSRkZGUpMTFTfvn11//3369y5c2aX1GJFRUWSpPbt20uSduzYoaqqqnr3sV+/furatWurvI/fvL46K1asUIcOHTRo0CAtXLhQZWVlZpR3RVwul1auXKnS0lKlp6cH3b2TLr7GOsFw/+bOnatp06bVu1+S+f8NtrmFM73t7NmzcrlcSkpKqrc/KSlJhw4dMqkq7xo9erSWL1+uvn37Kjc3V4888oiuueYa7du3T7GxsWaX51V5eXmS1OD9rHsvGEydOlU333yzunfvrqNHj+pXv/qVrr/+em3evFmhoaFml9csbrdbCxYs0Lhx4zRo0CBJNfcxIiJCcXFx9Y5tjfexoeuTpO9///tKS0tTp06d9MUXX+gXv/iFMjMz9eabb5pYbdPt3btX6enpqqioUExMjFavXq0BAwZo9+7dQXPvLnWNUuu/f5K0cuVK7dy5U9u2bbvoPbP/GyTc4LKuv/56z/eDBw/W6NGjlZaWptdff10/+MEPTKwMLXX77bd7vr/qqqs0ePBg9ezZUxkZGZo4caKJlTXf3LlztW/fvlY9Dqwxl7q+H/7wh57vr7rqKqWkpGjixIk6evSoevbs6e8ym61v377avXu3ioqKtGrVKs2ZM0ebNm0yuyyvutQ1DhgwoNXfv+zsbM2fP18bNmxQZGSk2eVchG6pK9ShQweFhoZeNAI8Pz9fycnJJlXlW3FxcerTp4+OHDlidileV3fP2tL9lKQePXqoQ4cOre6ezps3T2+//bY+/PBDdenSxbM/OTlZlZWVKiwsrHd8a7uPl7q+howePVqSWs09jIiIUK9evTR8+HAtWbJEQ4YM0dNPPx0090669DU2pLXdvx07dqigoEDDhg1TWFiYwsLCtGnTJj3zzDMKCwtTUlKSqfeRcHOFIiIiNHz4cG3cuNGzz+12a+PGjfX6VoNJSUmJjh49qpSUFLNL8bru3bsrOTm53v10OBzaunVr0N5PSTp16pTOnTvXau6pYRiaN2+eVq9erQ8++EDdu3ev9/7w4cMVHh5e7z5mZmYqKyurVdzHy11fQ3bv3i1JreYefpPb7ZbT6Wz1964xddfYkNZ2/yZOnKi9e/dq9+7dnm3EiBGaNWuW53tT76PPhyy3AStXrjSsVquxfPly48CBA8YPf/hDIy4uzsjLyzO7NK/46U9/amRkZBjHjx83Pv30U2PSpElGhw4djIKCArNLa5Hi4mJj165dxq5duwxJxv/93/8Zu3btMk6ePGkYhmEsXbrUiIuLM9auXWt88cUXxvTp043u3bsb5eXlJlfedI1dY3FxsfGzn/3M2Lx5s3H8+HHj/fffN4YNG2b07t3bqKioMLv0Jrn//vsNu91uZGRkGLm5uZ6trKzMc8x9991ndO3a1fjggw+M7du3G+np6UZ6erqJVTfd5a7vyJEjxuLFi43t27cbx48fN9auXWv06NHDuPbaa02uvGl++ctfGps2bTKOHz9ufPHFF8Yvf/lLw2KxGOvXrzcMo3XfuzqNXWNrv3+X8s0ZYGbeR8KNl/zhD38wunbtakRERBijRo0ytmzZYnZJXvO9733PSElJMSIiIozOnTsb3/ve94wjR46YXVaLffjhh4aki7Y5c+YYhlEzHfzBBx80kpKSDKvVakycONHIzMw0t+hmauway8rKjMmTJxsdO3Y0wsPDjbS0NOPee+9tVWG8oWuTZLz00kueY8rLy40HHnjAiI+PN6KiooybbrrJyM3NNa/oZrjc9WVlZRnXXnut0b59e8NqtRq9evUyfv7znxtFRUXmFt5Ed999t5GWlmZEREQYHTt2NCZOnOgJNobRuu9dncausbXfv0v5Zrgx8z5aDMMwfN8+BAAA4B+MuQEAAEGFcAMAAIIK4QYAAAQVwg0AAAgqhBsAABBUCDcAACCoEG4AAEBQIdwAAICgQrgBEJDOnDmj+++/X127dpXValVycrKmTJmiTz/9VJJksVi0Zs0ac4sEEJDCzC4AABpyyy23qLKyUn/729/Uo0cP5efna+PGjTp37pzZpQEIcCy/ACDgFBYWKj4+XhkZGRo/fvxF73fr1k0nT570vE5LS9OJEyckSWvXrtUjjzyiAwcOqFOnTpozZ45+/etfKyys5v/LWSwW/elPf9K//vUvZWRkKCUlRU888YRuvfVWv1wbAN+jWwpAwImJiVFMTIzWrFkjp9N50fvbtm2TJL300kvKzc31vP744481e/ZszZ8/XwcOHNDzzz+v5cuX67HHHqt3/oMPPqhbbrlFe/bs0axZs3T77bfr4MGDvr8wAH5Byw2AgPTPf/5T9957r8rLyzVs2DCNHz9et99+uwYPHiyppgVm9erVmjFjhuecSZMmaeLEiVq4cKFn36uvvqr//u//Vk5Ojue8++67T8uWLfMcM2bMGA0bNkx/+tOf/HNxAHyKlhsAAemWW25RTk6O/vWvf2nq1KnKyMjQsGHDtHz58kues2fPHi1evNjT8hMTE6N7771Xubm5Kisr8xyXnp5e77z09HRaboAgwoBiAAErMjJS1113na677jo9+OCDuueee/Twww/rzjvvbPD4kpISPfLII7r55psb/CwAbQMtNwBajQEDBqi0tFSSFB4eLpfLVe/9YcOGKTMzU7169bpoCwn56p+7LVu21Dtvy5Yt6t+/v+8vAIBf0HIDIOCcO3dOt912m+6++24NHjxYsbGx2r59u5544glNnz5dUs2MqY0bN2rcuHGyWq2Kj4/XQw89pO985zvq2rWrbr31VoWEhGjPnj3at2+ffvvb33o+/4033tCIESP0//7f/9OKFSv0+eef6y9/+YtZlwvAyxhQDCDgOJ1OLVq0SOvXr9fRo0dVVVWl1NRU3XbbbfrVr36ldu3a6a233tJPfvITnThxQp07d/ZMBX/vvfe0ePFi7dq1S+Hh4erXr5/uuece3XvvvZJqBhT/8Y9/1Jo1a/TRRx8pJSVFjz/+uL773e+aeMUAvIlwA6BNaWiWFYDgwpgbAAAQVAg3AAAgqDCgGECbQk88EPxouQEAAEGFcAMAAIIK4QYAAAQVwg0AAAgqhBsAABBUCDcAACCoEG4AAEBQIdwAAICgQrgBAABB5f8Hi1aIxHll510AAAAASUVORK5CYII=\n", "text/plain": [ "
" ] @@ -1099,14 +1101,22 @@ "base_uri": "https://localhost:8080/" }, "id": "EkoFGCgSZ1yz", - "outputId": "3467b8ba-ce05-42f0-fb89-75922cc91e31" + "outputId": "13434f77-2d73-4176-db24-7c0b2e232fe6" }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:[process=0][thread=MainThread] Skipped cross-host ArrayMetadata validation because only one process is found: process_index=0.\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "_CHECKPOINT_METADATA d manifest.ocdbt _METADATA ocdbt.process_0 _sharding\n" + "array_metadatas d\t\t _METADATA _sharding\n", + "_CHECKPOINT_METADATA manifest.ocdbt ocdbt.process_0\n" ] } ], @@ -1124,35 +1134,62 @@ }, { "cell_type": "markdown", - "id": "3813cbf2", - "metadata": {}, + "metadata": { + "id": "3813cbf2" + }, "source": [ "## Profiling for hyperparameter tuning" ] }, { "cell_type": "code", - "execution_count": null, - "id": "b5d933c6", - "metadata": {}, - "outputs": [], + "execution_count": 14, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "b5d933c6", + "outputId": "b1233c24-0832-49f8-8568-e4ccbd73ca83" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.8/12.8 MB\u001b[0m \u001b[31m63.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m644.9/644.9 MB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.5/5.5 MB\u001b[0m \u001b[31m75.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m104.8/104.8 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.5/57.5 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.5/24.5 MB\u001b[0m \u001b[31m79.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m294.6/294.6 kB\u001b[0m \u001b[31m19.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.6/6.6 MB\u001b[0m \u001b[31m111.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.1/5.1 MB\u001b[0m \u001b[31m88.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m18.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.5/72.5 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h" + ] + } + ], "source": [ "!pip install -Uq tensorboard-plugin-profile tensorflow tensorboard" ] }, { "cell_type": "markdown", - "id": "2ac5fc4d", - "metadata": {}, + "metadata": { + "id": "2ac5fc4d" + }, "source": [ "Load the tensorboard colab extension." ] }, { "cell_type": "code", - "execution_count": null, - "id": "74f0c212", - "metadata": {}, + "execution_count": 15, + "metadata": { + "id": "74f0c212" + }, "outputs": [], "source": [ "%load_ext tensorboard" @@ -1160,17 +1197,19 @@ }, { "cell_type": "markdown", - "id": "17c6131f", - "metadata": {}, + "metadata": { + "id": "17c6131f" + }, "source": [ "As we're going to be running this model a number of times, we need some scaffolding to more easily compare our work. For a baseline, we'll need to perform some warmup to guarantee that our code is JIT'd and that our TPUs are warm. For improved comparability, we'll only start tracing after we've finished warmup." ] }, { "cell_type": "code", - "execution_count": null, - "id": "ddfd576e", - "metadata": {}, + "execution_count": 16, + "metadata": { + "id": "ddfd576e" + }, "outputs": [], "source": [ "trace_dir = \"/tmp/jax-trace/\"\n", @@ -1195,17 +1234,19 @@ }, { "cell_type": "markdown", - "id": "de70f5b7", - "metadata": {}, + "metadata": { + "id": "de70f5b7" + }, "source": [ "Now we'll perform some traces to compare results of different batch sizes. This will take several minutes as we need to reprocess our input data to prepare new batches each time." ] }, { "cell_type": "code", - "execution_count": null, - "id": "bc9452a6", - "metadata": {}, + "execution_count": 17, + "metadata": { + "id": "bc9452a6" + }, "outputs": [], "source": [ "trace_dir = \"/tmp/jax-trace-batch-comparison/\"\n", @@ -1221,8 +1262,9 @@ }, { "cell_type": "markdown", - "id": "ea379965", - "metadata": {}, + "metadata": { + "id": "ea379965" + }, "source": [ "Run Tensorboard with the Profiler Plugin to compare our runs. Runs are listed in order from newest to oldest, so the top run in the list will be have `batch_size = 256`.\n", "\n", @@ -1234,8 +1276,9 @@ { "cell_type": "code", "execution_count": null, - "id": "b86c565a", - "metadata": {}, + "metadata": { + "id": "b86c565a" + }, "outputs": [], "source": [ "%tensorboard --logdir=$trace_dir" @@ -1243,8 +1286,9 @@ }, { "cell_type": "markdown", - "id": "657967a5", - "metadata": {}, + "metadata": { + "id": "657967a5" + }, "source": [ "Next, we can explore alternative parallelism methods. In cell #4, we used 4-way data parallel and 2-way tensor parallel. 8-way data parallel is another popular way. Let's compare results between them. To switch to 8-way data parallel, we'll replace the `Mesh` definition with:\n", "\n", @@ -1257,9 +1301,10 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "80daa8dc", - "metadata": {}, + "execution_count": 18, + "metadata": { + "id": "80daa8dc" + }, "outputs": [], "source": [ "trace_dir = \"/tmp/jax-trace-parallelism-comparison/\"\n", @@ -1273,8 +1318,9 @@ }, { "cell_type": "markdown", - "id": "ad96e72b", - "metadata": {}, + "metadata": { + "id": "ad96e72b" + }, "source": [ "Once again we'll run tensorboard.\n", "\n", @@ -1286,8 +1332,9 @@ { "cell_type": "code", "execution_count": null, - "id": "780e9c72", - "metadata": {}, + "metadata": { + "id": "780e9c72" + }, "outputs": [], "source": [ "%tensorboard --logdir=$trace_dir" @@ -1295,8 +1342,9 @@ }, { "cell_type": "markdown", - "id": "deca486e", - "metadata": {}, + "metadata": { + "id": "deca486e" + }, "source": [ "By changing hyperparameters and comparing profiles, we're able to gain significant insights into our bottlenecks and limitations. These are just two examples of hyperparameters to tune, but plenty more of them will have significant effects on training speed and resource utilization." ] diff --git a/docs/source/JAX_for_LLM_pretraining.md b/docs/source/JAX_for_LLM_pretraining.md index 267729b..6eff7ab 100644 --- a/docs/source/JAX_for_LLM_pretraining.md +++ b/docs/source/JAX_for_LLM_pretraining.md @@ -40,7 +40,7 @@ JAX installation is covered in [this guide](https://jax.readthedocs.io/en/latest colab: base_uri: https://localhost:8080/ id: 6zMsOIc7ouCO -outputId: 037d56a9-b18f-4504-f80a-3a4fa2945068 +outputId: ad486e3b-dd63-405f-d786-79b0b6d60cbd --- !pip install -Uq tiktoken grain matplotlib ``` @@ -56,7 +56,7 @@ Check the available JAX devices, or [`jax.Device`](https://jax.readthedocs.io/en colab: base_uri: https://localhost:8080/ id: LS9sQEY3n0mB -outputId: 9ffcf3a6-20ef-4f80-b006-f5d3c5644a15 +outputId: c18a63d0-696e-4c93-f8d5-8c7942045005 --- import jax jax.devices() @@ -71,7 +71,7 @@ Get the [TinyStories dataset from Hugging Face](https://huggingface.co/datasets/ colab: base_uri: https://localhost:8080/ id: wUjQsgQEmI1N -outputId: e6eff24e-5578-4277-a0f9-24e27bd91ee0 +outputId: 8a75571a-8339-4f1a-958b-5805aa285bb9 --- !wget https://huggingface.co/datasets/roneneldan/TinyStories/resolve/main/TinyStories-train.txt?download=true -O TinyStories-train.txt ``` @@ -318,48 +318,31 @@ class MiniGPT(nnx.Module): outputs = self.output_layer(x) return outputs - # Text generation. - def generate_text(self, max_tokens: int, start_tokens: [int], top_k=10): - # Sample the next token from a probability distribution based on - # `logits` and `tok_k` (top-k) sampling strategy. - def sample_from(logits): - logits, indices = jax.lax.top_k(logits, k=top_k) - # Convert logits to probabilities (using `flax.nnx.softmax`). - logits = nnx.softmax(logits) - return jax.random.choice(jax.random.PRNGKey(0), indices, p=logits) - - # Generate text one token at a time until the maximum token limit is reached (`maxlen`). - def generate_step(start_tokens): - pad_len = maxlen - len(start_tokens) - # Index of the last token in the current sequence. - sample_index = len(start_tokens) - 1 - # If the input is longer than `maxlen`, then truncate it. - if pad_len < 0: - x = jnp.array(start_tokens[:maxlen]) - sample_index = maxlen - 1 - # If the input is shorter than `maxlen`, then pad it (`pad_len`). - elif pad_len > 0: - x = jnp.array(start_tokens + [0] * pad_len) - else: - x = jnp.array(start_tokens) - - # Add a batch dimension. - x = x[None, :] - logits = self(x) - next_token = sample_from(logits[0][sample_index]) - return next_token - - # Store generated tokens. + @nnx.jit + def sample_from(self, logits): + logits, indices = jax.lax.top_k(logits, k=top_k) + logits = nnx.softmax(logits) + return jax.random.choice(jax.random.PRNGKey(0), indices, p=logits) + + @nnx.jit + def generate_step(self, padded_tokens, sample_index): + logits = self(padded_tokens) + next_token = self.sample_from(logits[0][sample_index]) + return next_token + + def generate_text(self, max_tokens, start_tokens): generated = [] - # Generate tokens until the end-of-text token is encountered or the maximum token limit is reached. - for _ in range(max_tokens): - next_token = generate_step(start_tokens + generated) - # Truncate whatever is after '<|endoftext|>' (stop word) + print(tokenizer.decode(start_tokens), flush=True, end='') + for i in range(max_tokens): + sample_index = len(start_tokens) + len(generated) - 1 + + padded_tokens = jnp.array((start_tokens + generated + [0] * (maxlen - len(start_tokens) - len(generated))))[None, :] + next_token = int(self.generate_step(padded_tokens, sample_index)) if next_token == tokenizer.encode('<|endoftext|>', allowed_special={'<|endoftext|>'})[0]: - # Stop text generation if the end-of-text token is encountered. break - generated.append(int(next_token)) - # Decode the generated token IDs into text. + generated.append(next_token) + # decode and print next_token + print(tokenizer.decode([next_token]), flush=True, end='') return tokenizer.decode(start_tokens + generated) # Creates the miniGPT model with 4 transformer blocks. @@ -382,6 +365,7 @@ num_heads = 8 feed_forward_dim = 256 batch_size = 256 # You can set a bigger batch size if you use Kaggle's Cloud TPU. num_epochs = 1 +top_k = 10 ``` +++ {"id": "mI1ci-HyMspJ"} @@ -473,7 +457,7 @@ We are also using the `jax.vmap` transformation to produce the target sequences colab: base_uri: https://localhost:8080/ id: Ysl6CsfENeJN -outputId: 5dd06dca-f030-4927-a9b6-35d412da535c +outputId: b713cc63-a4ff-4fea-96ea-c234348e0ea4 --- model = create_model(rngs=nnx.Rngs(0)) optimizer = nnx.Optimizer(model, optax.adam(1e-3)) @@ -484,11 +468,10 @@ rng = jax.random.PRNGKey(0) start_prompt = "Once upon a time" start_tokens = tokenizer.encode(start_prompt)[:maxlen] +print(f"Initial generated text:") generated_text = model.generate_text( maxlen, start_tokens ) -print(f"Initial generated text:\n{generated_text}\n") - metrics_history = { 'train_loss': [], @@ -512,20 +495,21 @@ for epoch in range(num_epochs): metrics.reset() elapsed_time = time.time() - start_time - print(f"Step {step + 1}, Loss: {metrics_history['train_loss'][-1]}, Elapsed Time: {elapsed_time:.2f} seconds") + print(f"\n\nStep {step + 1}, Loss: {metrics_history['train_loss'][-1]}, Elapsed Time: {elapsed_time:.2f} seconds") start_time = time.time() + print(f"Generated text:") generated_text = model.generate_text( maxlen, start_tokens ) - print(f"Generated text:\n{generated_text}\n") + step += 1 # Final text generation +print(f"Final generated text:") generated_text = model.generate_text( maxlen, start_tokens ) -print(f"Final generated text:\n{generated_text}") ``` +++ {"id": "thaLs6TD0lt5"} @@ -538,7 +522,7 @@ colab: base_uri: https://localhost:8080/ height: 472 id: B6Eg1Cz2y_iP -outputId: 7cafe711-1ae4-4eb9-fd37-e1bde54cbfc5 +outputId: fb96b456-23a8-448f-ebc6-23d807a626d2 --- import matplotlib.pyplot as plt plt.plot(metrics_history['train_loss']) @@ -563,7 +547,7 @@ Save the model checkpoint. colab: base_uri: https://localhost:8080/ id: EkoFGCgSZ1yz -outputId: 3467b8ba-ce05-42f0-fb89-75922cc91e31 +outputId: 13434f77-2d73-4176-db24-7c0b2e232fe6 --- import orbax.checkpoint as orbax @@ -576,21 +560,37 @@ checkpointer.save('/content/save', state) !ls /content/save/ ``` ++++ {"id": "3813cbf2"} + ## Profiling for hyperparameter tuning ```{code-cell} +--- +colab: + base_uri: https://localhost:8080/ +id: b5d933c6 +outputId: b1233c24-0832-49f8-8568-e4ccbd73ca83 +--- !pip install -Uq tensorboard-plugin-profile tensorflow tensorboard ``` ++++ {"id": "2ac5fc4d"} + Load the tensorboard colab extension. ```{code-cell} +:id: 74f0c212 + %load_ext tensorboard ``` ++++ {"id": "17c6131f"} + As we're going to be running this model a number of times, we need some scaffolding to more easily compare our work. For a baseline, we'll need to perform some warmup to guarantee that our code is JIT'd and that our TPUs are warm. For improved comparability, we'll only start tracing after we've finished warmup. ```{code-cell} +:id: ddfd576e + trace_dir = "/tmp/jax-trace/" def loop_step(batch, step): @@ -611,9 +611,13 @@ def generate_trace(): jax.profiler.stop_trace() ``` ++++ {"id": "de70f5b7"} + Now we'll perform some traces to compare results of different batch sizes. This will take several minutes as we need to reprocess our input data to prepare new batches each time. ```{code-cell} +:id: bc9452a6 + trace_dir = "/tmp/jax-trace-batch-comparison/" batch_size = 64 @@ -625,6 +629,8 @@ text_dl = iter(load_and_preprocess_data('TinyStories-train.txt', batch_size, max generate_trace() ``` ++++ {"id": "ea379965"} + Run Tensorboard with the Profiler Plugin to compare our runs. Runs are listed in order from newest to oldest, so the top run in the list will be have `batch_size = 256`. The key metrics to focus on here for this hyperparameter are FLOPS Utilization and Average Step Time. @@ -632,9 +638,13 @@ The key metrics to focus on here for this hyperparameter are FLOPS Utilization a In general, we want to maximize FLOPS Utilization while minimizing the step time per training example. In this case, we can see that increasing the batch size from 64 -> 256 achieves both of those. FLOPS increases from 16% to 27%. Average Step Time increase from 100ms to 260ms, however we increased our batch size by 300%. This means we move from 1.5ms per training example to 1.02ms per training example. ```{code-cell} +:id: b86c565a + %tensorboard --logdir=$trace_dir ``` ++++ {"id": "657967a5"} + Next, we can explore alternative parallelism methods. In cell #4, we used 4-way data parallel and 2-way tensor parallel. 8-way data parallel is another popular way. Let's compare results between them. To switch to 8-way data parallel, we'll replace the `Mesh` definition with: `mesh = Mesh(mesh_utils.create_device_mesh((8, 1)), ('batch', 'model'))` @@ -644,6 +654,8 @@ JAX will automatically figure out how to shard the model and data to use the new How simple and powerful is this! And that's the beauty of JAX automatic parallelism. ```{code-cell} +:id: 80daa8dc + trace_dir = "/tmp/jax-trace-parallelism-comparison/" mesh = Mesh(mesh_utils.create_device_mesh((4, 2)), ('batch', 'model')) @@ -653,6 +665,8 @@ mesh = Mesh(mesh_utils.create_device_mesh((8, 1)), ('batch', 'model')) generate_trace() ``` ++++ {"id": "ad96e72b"} + Once again we'll run tensorboard. Looking at the results, we see that the step times are nearly the same, however the FLOPS Utilization is at 13% for 8-way data parallelism compared to 27% or 4-way data parallelism. @@ -660,7 +674,11 @@ Looking at the results, we see that the step times are nearly the same, however By looking at the Trace Viewer tool and looking under each TPU's ops, we can see that the TPUs spend a large amount of time idle while waiting for the host, as well as spending a good amount of time in `reduce_sum` operations. ```{code-cell} +:id: 780e9c72 + %tensorboard --logdir=$trace_dir ``` ++++ {"id": "deca486e"} + By changing hyperparameters and comparing profiles, we're able to gain significant insights into our bottlenecks and limitations. These are just two examples of hyperparameters to tune, but plenty more of them will have significant effects on training speed and resource utilization.