diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f5fb7ad..564e46da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,8 +17,6 @@ jobs: fail-fast: false matrix: version: - - '1.3' - - '1.4' - '1' - 'nightly' os: diff --git a/Project.toml b/Project.toml index 884a94e3..d34192ed 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "MLJFlux" uuid = "094fc8d1-fd35-5302-93ea-dabda2abf845" authors = ["Anthony D. Blaom ", "Ayush Shridhar "] -version = "0.1.11" +version = "0.1.12" [deps] CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" @@ -15,15 +15,15 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] -CategoricalArrays = "^0.10" -ColorTypes = "^0.10.3, 0.11" -ComputationalResources = "^0.3.2" -Flux = "^0.10.4, ^0.11, 0.12" -LossFunctions = "^0.5, ^0.6" -MLJModelInterface = "^0.4.1, 1" -ProgressMeter = "^1.1" -Tables = "^1.0" -julia = "^1.3" +CategoricalArrays = "0.10" +ColorTypes = "0.10.3, 0.11" +ComputationalResources = "0.3.2" +Flux = "0.10.4, 0.11, 0.12" +LossFunctions = "0.5, 0.6" +MLJModelInterface = "1.1" +ProgressMeter = "1.1" +Tables = "1.0" +julia = "1.6" [extras] LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" diff --git a/examples/iris/iris.ipynb b/examples/iris/iris.ipynb index f7f62df3..e37433c0 100644 --- a/examples/iris/iris.ipynb +++ b/examples/iris/iris.ipynb @@ -13,7 +13,7 @@ "name": "stdout", "output_type": "stream", "text": [ - " Activating environment at `~/Dropbox/Julia7/MLJ/MLJFlux/examples/iris/Project.toml`\n" + " Activating environment at `~/Dropbox/Julia7/MLJ/MLJFlux/examples/iris/Project.toml`\n" ] } ], @@ -21,8 +21,22 @@ "source": [ "using Pkg\n", "Pkg.activate(@__DIR__)\n", - "Pkg.instantiate()\n", - "\n", + "Pkg.instantiate()" + ], + "metadata": {}, + "execution_count": 1 + }, + { + "cell_type": "markdown", + "source": [ + "**Julia version** is assumed to be 1.6.*" + ], + "metadata": {} + }, + { + "outputs": [], + "cell_type": "code", + "source": [ "using MLJ\n", "using Flux\n", "import RDatasets\n", @@ -36,7 +50,7 @@ "pyplot(size=(600, 300*(sqrt(5)-1)));" ], "metadata": {}, - "execution_count": 1 + "execution_count": 2 }, { "cell_type": "markdown", @@ -68,17 +82,17 @@ "output_type": "stream", "text": [ "┌ Info: For silent loading, specify `verbosity=0`. \n", - "└ @ Main.##407 /Users/anthony/.julia/packages/MLJModels/66QJr/src/loading.jl:168\n", + "└ @ Main.##411 /Users/anthony/.julia/packages/MLJModels/66QJr/src/loading.jl:168\n", "import MLJFlux ✔\n" ] }, { "output_type": "execute_result", "data": { - "text/plain": "NeuralNetworkClassifier(\n builder = Short(\n n_hidden = 0,\n dropout = 0.5,\n σ = NNlib.σ),\n finaliser = NNlib.softmax,\n optimiser = ADAM(0.001, (0.9, 0.999), IdDict{Any,Any}()),\n loss = Flux.Losses.crossentropy,\n epochs = 10,\n batch_size = 1,\n lambda = 0.0,\n alpha = 0.0,\n optimiser_changes_trigger_retraining = false,\n acceleration = CPU1{Nothing}(nothing)) @443" + "text/plain": "NeuralNetworkClassifier(\n builder = Short(\n n_hidden = 0,\n dropout = 0.5,\n σ = NNlib.σ),\n finaliser = NNlib.softmax,\n optimiser = ADAM(0.001, (0.9, 0.999), IdDict{Any, Any}()),\n loss = Flux.Losses.crossentropy,\n epochs = 10,\n batch_size = 1,\n lambda = 0.0,\n alpha = 0.0,\n optimiser_changes_trigger_retraining = false,\n acceleration = CPU1{Nothing}(nothing)) @933" }, "metadata": {}, - "execution_count": 2 + "execution_count": 3 } ], "cell_type": "code", @@ -89,7 +103,7 @@ "clf = NeuralNetworkClassifier()" ], "metadata": {}, - "execution_count": 2 + "execution_count": 3 }, { "cell_type": "markdown", @@ -104,9 +118,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "┌ Info: Training Machine{NeuralNetworkClassifier{Short,…},…} @158.\n", + "┌ Info: Training Machine{NeuralNetworkClassifier{Short,…},…} @886.\n", "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/4DmTL/src/machines.jl:341\n", - "\rOptimising neural net: 9%[==> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 18%[====> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 27%[======> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 36%[=========> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 45%[===========> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 55%[=============> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 64%[===============> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 73%[==================> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 82%[====================> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 91%[======================> ] ETA: 0:00:00\u001b[K\rOptimising neural net:100%[=========================] Time: 0:00:00\u001b[K\n" + "\rOptimising neural net: 9%[==> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 27%[======> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 36%[=========> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 45%[===========> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 55%[=============> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 64%[===============> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 73%[==================> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 82%[====================> ] ETA: 0:00:00\u001b[K\rOptimising neural net: 91%[======================> ] ETA: 0:00:00\u001b[K\rOptimising neural net:100%[=========================] Time: 0:00:00\u001b[K\n" ] }, { @@ -115,7 +129,7 @@ "text/plain": "0.8993467f0" }, "metadata": {}, - "execution_count": 3 + "execution_count": 4 } ], "cell_type": "code", @@ -127,7 +141,7 @@ "training_loss = cross_entropy(predict(mach, X), y) |> mean" ], "metadata": {}, - "execution_count": 3 + "execution_count": 4 }, { "cell_type": "markdown", @@ -142,7 +156,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "┌ Info: Updating Machine{NeuralNetworkClassifier{Short,…},…} @158.\n", + "┌ Info: Updating Machine{NeuralNetworkClassifier{Short,…},…} @886.\n", "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/4DmTL/src/machines.jl:342\n", "┌ Info: Loss is 0.853\n", "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/wj7HX/src/core.jl:122\n", @@ -165,7 +179,7 @@ "fit!(mach, verbosity=2);" ], "metadata": {}, - "execution_count": 4 + "execution_count": 5 }, { "outputs": [ @@ -175,7 +189,7 @@ "text/plain": "0.7076617f0" }, "metadata": {}, - "execution_count": 5 + "execution_count": 6 } ], "cell_type": "code", @@ -183,7 +197,7 @@ "training_loss = cross_entropy(predict(mach, X), y) |> mean" ], "metadata": {}, - "execution_count": 5 + "execution_count": 6 }, { "cell_type": "markdown", @@ -200,7 +214,7 @@ "text/plain": "Chain(Chain(Dense(4, 3, σ), Dropout(0.5), Dense(3, 3)), softmax)" }, "metadata": {}, - "execution_count": 6 + "execution_count": 7 } ], "cell_type": "code", @@ -208,7 +222,7 @@ "chain = fitted_params(mach).chain" ], "metadata": {}, - "execution_count": 6 + "execution_count": 7 }, { "cell_type": "markdown", @@ -223,20 +237,20 @@ "name": "stdout", "output_type": "stream", "text": [ - "┌ Info: Training Machine{ProbabilisticTunedModel{Grid,…},…} @663.\n", + "┌ Info: Training Machine{ProbabilisticTunedModel{Grid,…},…} @380.\n", "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/4DmTL/src/machines.jl:341\n", "┌ Info: Attempting to evaluate 25 models.\n", "└ @ MLJTuning /Users/anthony/.julia/packages/MLJTuning/wBJ80/src/tuned_models.jl:566\n", - "\rEvaluating over 25 metamodels: 0%[> ] ETA: N/A\u001b[K\rEvaluating over 25 metamodels: 4%[=> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 8%[==> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 12%[===> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 16%[====> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 20%[=====> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 24%[======> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 28%[=======> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 32%[========> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 36%[=========> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 40%[==========> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 44%[===========> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 48%[============> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 52%[=============> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 56%[==============> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 60%[===============> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 64%[================> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 68%[=================> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 72%[==================> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 76%[===================> ] ETA: 0:00:01\u001b[K\rEvaluating over 25 metamodels: 80%[====================> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 84%[=====================> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 88%[======================> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 92%[=======================> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 96%[========================>] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 100%[=========================] Time: 0:00:04\u001b[K\n" + "\rEvaluating over 25 metamodels: 0%[> ] ETA: N/A\u001b[K\rEvaluating over 25 metamodels: 4%[=> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 8%[==> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 12%[===> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 16%[====> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 20%[=====> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 24%[======> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 28%[=======> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 32%[========> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 36%[=========> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 40%[==========> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 44%[===========> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 48%[============> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 52%[=============> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 56%[==============> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 60%[===============> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 64%[================> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 68%[=================> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 72%[==================> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 76%[===================> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 80%[====================> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 84%[=====================> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 88%[======================> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 92%[=======================> ] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 96%[========================>] ETA: 0:00:00\u001b[K\rEvaluating over 25 metamodels: 100%[=========================] Time: 0:00:03\u001b[K\n" ] }, { "output_type": "execute_result", "data": { "text/plain": "Plot{Plots.PyPlotBackend() n=1}", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAFyCAYAAAApuaQRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3xUVd7H8c+59yb0BGLoJISW0ATpRbp0RJCAlRUUAQuydl0exc5acVUsqFgRV6QjKEgRUekREKkiIUFYkGKl3nvn+SOPPBshSMJkkkm+79fL15rcOzO/cJzsl3PO/I4JBAIBRERERCRorLwuQERERKSgCeuAdfjwYVJSUjh8+HBelyIiIiJyUlgHrM2bN9OkSRM2b96c4+f4+eefg1iRhJrGL7xp/MKfxjC8afxyT1gHrGDwPC+vS5BzoPELbxq/8KcxDG8av9xT6AOWiIiISLApYImIiIgEmQKWiIiISJA5eV2AiIiIhJ+0tDT279+f12XkidjYWOLj4894jwKWiIiIZEtaWhp16tQptG2SihcvzqZNm84YshSwREREJFv279/P4cOHmThxInXq1MnrckJq06ZNDBw4kP379ytgZSUQCLDrsCEmJq8rERERCT916tShcePGeV1GvlSoN7lP3RGg2bwSjFrl8fsJHckoIiIiwVGoA1bPeMPIxOOM/cYn6UOXf2/30dnXIiIicq4KdcAq7hj+Ue84G/s7NI01XLnIo8NHHusPKGSJiIhIzhXqgPWH6lGGGV0dPulus/dIgEbTXUZ86XHwqIKWiIiIZJ8C1n/pFmexPtnhieYWb2/zSZzs8uomH89X0BIREZGzp4D1J5G24c4GNlsvc+gVbxj+hUfzmS5f7fXzujQREREJEwpYWahY3PB2B4cvL7EBuHCWxzWLXfYc1myWiIhIuEpNTaVDhw5ER0fTtGnTXHsdBay/0Lq8xco+DuPb2MxND5A02eXp9R7HPQUtERGRcBMVFcWjjz7KpEmTcvV1FLDOgm0ZhtWx2Ha5w6BaFves9Gkw1WVeupYNRURE8qOnnnqK4cOHn/z6p59+IjY2FoA2bdpQokSJXH39Qt3JPbvKFDG8cKHN0NoWt3zl0f0Tjz5Vfca2tKkeZfK6PBERkXxlz+EAe3LhuMKKxTO28pzJ0KFDSUpK4sknnyQ6OpoJEybQp08fYkJ0fIsCVg40OM/w2cU2H3wf4M4VHnWnuNzdwOLeCyyKOwpaIiIiAOM3+TyUEvzVngcaWzzYxD7jPaVLlyY5OZm33nqLkSNH8vLLL/Phhx8GvZasKGDlkDGGK2oYescbxqz1eWKdz1tbfZ5padO/msEYBS0RESnchtexuKRq8HcjVSx+dveNHDmSvn37UqNGDcqXL0+jRo2CXktWFLDOUYkIw2PNbK5NtLhtucdlCz06VjS80NqmXoxCloiIFF4Vi5uzDkO5oXbt2iQkJHDjjTfy5JNPhvS1tck9SGpGG2Z3c5jTzWbX7wEaTnO5dZnHT8f0aUMREZG8MnToUFzXpX///gAcO3aMKlWqMGDAANavX0+VKlX4xz/+EfTX1QxWkPWMt7iosuFf3/g88rXPpO98/tnM5tokg6VlQxERkZBauHAhN910ExEREQAUKVKEXbt25frragYrFxSxDfdcYLPlMoculQ3XL/VoOdNjxT61dRAREQmF3bt3U7t2bdauXcutt94a8tdXwMpFlUsY3uvksLS3zQk/QMuZHtcucdmrbvAiIiK5qlKlSmzevJmvvvqKUqVKhfz1FbBCoE0Fi9V9HV660GLWzgCJk12e/cbjhA6RFhERKZAUsELEtgw31s04RPrqmhZ3rvBpONVlwQ9aNhQRESloFLBC7Lyihpfa2Kzu63BeEUOXuR7Jn7qk/qrZLBERkYJCASuPNIo1fN7b5r2ONsv3BajzoctDazyOuApaIiIi4U4BKw8ZY7iqpsXmAQ5/r2/x2FqfOh+6TNvhEwgoaImIiIQrBax8oFSk4fHmNhuSHeqWMSQv8Oj6scemQwpZIiIi4UiNRvORxNKGOd1s5qQFuHW5R4OpLiPrW4xubBEdqSalIiKSv2zatCmvSwi5s/2ZFbDyGWMMF1c1dK5sGPuNz2Nrfd77zufx5jbX1FI3eBERyXuxsbEUL16cgQMH5nUpeaJ48eLExsae8R4FrHyqqGMY1cjmb7Us7lrhce0Sj/GbDC+0tmhaViu7IiKSd+Lj49m0aRP79+/P61LyRGxsLPHx8We8RwErn4srafj3RQ431vW55SuP5jM8rkvyGdPMplwxzWaJiEjeiI+P/8uQUZhpKiRMtK9okXKpw/OtLabuyOgG//wGD1fd4EVERPIdBaww4liGEfUyusFfXt3i1mU+jaa5LN6tbvAiIiL5iQJWGCpbzDC+rc2qvg6lIgyd5nhctsAl7TfNZomIiOQHClhhrElZwxeX2LzTwWbpfwLUnuzyaIrHUXWDFxERyVMKWGHOMoa/1bLYcpnDzfUsHkrxqTvFZWaqusGLiIjkFQWsAiIq0vBUC5tv+jvUijL0/dSjxyceW35SyBIREQk1BawCpnZpwyc9bGZ0sdn6c4Dzp7rcvcLj1+MKWiIiIqGigFUAGWPok2DxbX+H+xtZjPvWJ3Gyy7vbtGwoIiISCgpYBVgxx3B/Y5tNAxzaVjBc85lHm9keKfsVskRERHKTAlYhULWUYXJnh4U9bX4+HqDpdJcblnrsP6qgJSIikhsUsAqRTpUtvu7n8Gwri39/n7Fs+OK36gYvIiISbApYhUyEZfh7/Yxu8P0SDLd85dNkusvne9QNXkREJFgUsAqpcsUMr7dzWNHXpqhtaP+Rx1WLXPYe1myWiIjIuVLAKuSalbVY1sfmzfY2n/4QoM4Ul3e26tOGIiIi50IBS7CMYXCixaYBDj3jDIOWZDQp3fmrQpaIiEhOKGDJSbFFDRM7OnzUzebbQwHqTXEZ962Hr9ksERGRbFHAklP0is9oUvq3Wha3fOXTbrbHZh25IyIictYUsOS0oiINL7ex+exim31HAjSc6jLma48TaukgIiLylxSw5IzaV7RYl+xw2/kWo9f4NJvuqhO8iIjIXwhpwBo5ciQJCQkYY9iwYUOW923bto3WrVuTmJhI8+bN2bhxYwirlD8r5hgeb26zoo8DQPMZLveu9DjiKmiJiIicTkgDVv/+/fniiy+oWrXqGe8bPnw4w4YNY+vWrdx9990MGTIkRBXKmTQpa1h1qcPDTSye/cbngmkuS9WgVERE5BQhDVjt2rWjSpUqZ7xn3759pKSkMHDgQACSk5PZsWMHqampIahQ/kqEZRjVyGZtP4fYooZ2H3nc9IXHL8c1myUiIvIHJ68L+LP09HQqVaqE42SUZowhPj6etLQ0EhISTvuYESNGEB0dTb9+/UhOTs7W6x06dOhcSy6UygMzL4QJ2yN4ZEMRZqa6jG10lC4VvZDWofELbxq/8KcxDG8av+CJiYnJ9HW+C1iQEar+2191FR83bhyNGzfO8ev9+Q9Fzt4958EVdQIM/8Ljiq+Kc3VNw79a2cQWNX/94CDR+IU3jV/40xiGN41f7sh3nyKMi4tj165duK4LZISr9PR04uPj87gyyUrVUoaPu9u81d5mbnqAuh+6fLBdx+2IiEjhle8CVrly5WjUqBETJ04EYOrUqSQkJGS5PCj5gzGGQYkWG/s7tK9ouGKRR99PPX74XSFLREQKn5AGrJtvvpkqVaqwa9cuOnfuTM2aNU9e69mzJ6tXrwZg/PjxjB8/nsTERB5//HEmTJgQyjLlHFQobviws8O0zjYr92XMZr22WbNZIiJSuJhAGP8/X0pKCk2aNGHNmjU53oN18OBBrT/nkkPHAty1wmPClgAdKxpebWtTMzq4e7M0fuFN4xf+NIbhTeOXe/LdEqEUHGWKGF5v5/BpT5vU3wI0mOryzHoPV8ftiIhIAaeAJbmuc2WLb5IdhtexuGuFT+tZHt8cVMgSEZGCSwFLQqJEhOHZVjZfXWLz+4kAjae5PLDG45inoCUiIgWPApaEVMvyFin9HEY1shjztU/jaS7L9+q4HRERKVgUsCTkitiGh5rYpPRzKBFhaD3L47ZlHr+f0GyWiIgUDApYkmfOjzEsu8TmqRYW4zf51J/isuAHzWaJiEj4U8CSPGVbhjsa2KxPdkgoZegy12PIEpdDxzSbJSIi4UsBS/KFmtGGRb1sXm1rM2VHRoPS6Ts0myUiIuFJAUvyDWMMQ2tbbBzg0Lycod8CjwELXP5zWLNZIiISXhSwJN+pXMIwo4vNB51sluwJUHeKy9tbddyOiIiEDwUsyZeMMVxWw2LTAIdecYbBSzw6zvFYuU/LhiIikv8pYEm+dl5Rw7sdHT7pbnPgaIAWMz36L3DZ8pNms0REJP9SwJKw0C3OYm0/h7fb26z6MUC9KS7Dl3rsORLcw6NFRESCQQFLwoZtGa5JtNgywOGpFhZTdvg0m1eCUas8flJbBxERyUcUsCTsFHUMt51v8/0VDjfWOs6/vvGp8YHLM+s9jroKWiIikvcUsCRsRUca/qfecbZf4XBZdYt7VvokTnZ5a6uP5ytoiYhI3lHAkrBXsbjh5TY2G/s7tCxnuHaJR8NpLrN3qrWDiIjkDQUsKTASSxsmd3ZY2demXFHDJfM92s72+PI/au0gIiKhpYAlBU6zshYLe9l80t3mdzdAm9kefea7fHtQs1kiIhIaClhSIBlj6BZnseZSh0kdbb45GKDBNJfrlrik/6agJSIiuUsBSwo0yxiurGmxeYDDv1pafJQWoNZkl7tWeBw8qqAlIiK5QwFLCoVI23BLfZvtlzvc29DilU0+1T9weXytx2G1dhARkSBTwJJCpVSk4cEmGUHrmloWo9f41PrA5bXNPq5aO4iISJAoYEmhVK6Y4fnWNpsGOLSvaBi21KP+FJdpO9TaQUREzp0ClhRqNaIMkzo5pFzqkFDKkLzAo9UsjyV71NpBRERyTgFLBGgUa/ikh8PCnjaeDx0+8uj1iVo7iIhIzihgifyXTpUtVva1+aCTzeafMlo7DF/q8Z/DCloiInL2FLBE/sQYw2U1LDYNcHimhcWUHT41P3B5JMXj9xMKWiIi8tcUsESyEGkbbj3f5rvLHW6sa/Ho1xmHSb+xRYdJi4jImSlgifyFMkUMT7Ww2TzAoV1Fw5DPPRpNd5m/SxvhRUTk9BSwRM5StSjD+50clvexiY4wdPvYo/vHLusPaDZLREQyU8ASyaYW5Sw+720zvYvN978EuGCay5AlLrt/V9ASEZEMClgiOWCMoW+CxbcDHJ5vbTFzZ8YZhw+s8fhNG+FFRAo9BSyRcxBhGUbUs9l+hcMt9SyeWJfxiUMdvSMiUrgpYIkEQXSk4fHmNlsGOHSunHH0TsOpLnPTdPSOiEhhpIAlEkRVSxkmdnRY1dembDFDr3keXeZ6rNVGeBGRQkUBSyQXNC1rsbiXzayuNrt+D9B4msugz1zSf1PQEhEpDBSwRHKJMYbeVS2+6e/w4oUWH6cHSJzs8sx6D1/LhiIiBZoClkgui7AMN9bN6Ah/U12LO1f49PrEY98RhSwRkYJKAUskRKIiDc+0tPmku03KgQANp7os/EHd4EVECiIFLJEQ6xZnsa6fw/kxhi5zPUat8jihlg4iIgWKApZIHqhQ3PBJD5vHm1s8tc6n3WyP1F8VskRECgoFLJE8YhnD3Q1tlva2+c/hjCN3PvxeS4YiIgWBApZIHmtZ3uLrfg7dqhguW+gxfKnHYVezWSIi4UwBSyQfKF3E8O9ONq+1tXl3m0/zGS4bDipkiYiEKwUskXzCGMP1tS1WX+pggGYzXMZv8nTUjohIGFLAEsln6pYxrOzrcG2ixQ1f+AxY6HHomEKWiEg4yTJgNW3alFdeeYWff/45lPWICFDMMbzUxmZqZ5uFP2RsgP9qrzbAi4iEiywDVmJiIrfffjuVKlXimmuuYcmSJaGsS0SAftUs1vZziCthaDfbY8zXHp56ZomI5HtZBqxJkyaxZ88ennrqKTZt2kTHjh2pWbMm//znP9m9e3coaxQp1KqWMnx2sc0/LrC4b7VP1489dv+ukCUikp+dcQ9WdHQ0N910E6tWrWLt2rVcfPHFjB07lqpVq9K7d29mzJiB72vZQiS3OZbhkaY2C3vZbP4pQMNpLk+v9/jluIKWiEh+dNab3KtUqUK1atUoX748nuexbds2kpOTqVWrFsuXL8/NGkXk/3SsZLEu2aF3vGHUKp+4SS53r/D4QTNaIiL5yl8GrHnz5nH55ZdTuXJlHnvsMbp27cq3337L5s2b2bp1K7Vq1eK6664LRa0iAsQWNbzR3mHHFQ431LEYv8kn4X2XQZ+5fKPeWSIi+UKWAWv06NHEx8fTs2dP9u/fz5tvvsmuXbsYO3YsderUAaBGjRo88MADbNmyJWQFi0iGyiUMT7SwSb/K4YnmFot2B2gw1aXHxy6LfvDVP0tEJA9lGbBee+01rr76arZu3crChQu54ooriIyMPOW+pKQk3njjjVwtUkSyFhVpuL2BzfdXOLzbwWb34QAXzfVoOsPl/e98XH3qUEQk5JysLqSnp+M4WV4+KSYmhkGDBgW1KBHJvgjLMLCW4eqahk9/CPDUep+rFnv8YxXcdr7FkCSLkhEmr8sUESkUskxQf4SrrVu3snLlSvbs2UPFihVp1qwZSUlJIStQRLLHGEPXKoauVSzWHgjw9HqPO5b7PLjG58a6FiPrWVQorqAlIpKbsgxYv/32G8OGDWPy5Mn4vk9ERAQnTpzAsiwGDBjAa6+9RsmSJUNZq4hk0wXnGSZ2dBjTLMBzG3xe+NbnmfU+T7bICFrGKGiJiOSGLPdg3XLLLXz00Ue8+uqrHDx4kGPHjnHw4EHGjx/PnDlzuOWWW0JZp4icg/iShmda2qRf6XBjXYtbl/lcucjjtxPanyUikhuyDFhTp07liSee4LrrrqN06dIAlC5dmiFDhvD4448zbdq0kBUpIsFRuojhX61sJl9kMyc9QPMZLpsOKWSJiARblgGraNGiVKtW7bTXqlevTkRERK4VJSK5a0B1i1V9M3YINJvhMnm7TmQQEQmmLAPWtddey8svv3xKL51AIMBLL73Etddem+vFiUjuqV3asLKvQ++qhssXedy6zOO4p9ksEZFgyHKTe0xMDCkpKdSqVYvevXtTrlw59u3bx+zZszl27Bht2rRh7NixQManlm677ba/fLFt27YxaNAg9u/fT+nSpXnrrbeoW7fuKffNmzePUaNG4fs+J06c4K677lIrCJFcUDLCMKmjTetyPrcv91n1Y4DJF9lULqHN7yIi58IEsmj3bFlnfUwhxhg8z/vL+zp16sQ111zD4MGDmTJlCs888wzLli3LdE8gECA2NpbFixfToEEDUlNTqV27Nj/++COlSpXKdG9KSgpNmjRhzZo1NG7c+Kzr/W8HDx4kJiYmR4+VvKfxC55le30GLPQ44cO/O9l0rHT2vwNySuMX/jSG4U3jl3uynMHy/eDuydi3bx8pKSnMnz8fgOTkZEaMGEFqaioJCQmn3P/TTz8B8Msvv3DeeedRpEiRLJ97xIgRREdH069fP5KTk7NV16FDh7J1v+QvGr/gSYqAhR0MQ1cWpfOcAPfVP87IxOPkZicHjV/40xiGN41f8Pw5qP51q/YgSU9Pp1KlSicbmBpjiI+PJy0tLVPAMsYwefJk+vXrR4kSJTh06BDTpk077TE9fxg3blyOZ7Dg1D8UCS8av+CJARZfEmD0Gp+H1xZh3a9FebuDTXRk7qUsjV/40xiGN41f7jjjGsCBAwd4+OGH6dKlC02aNKFLly488sgjHDhwIEcv9uemhqdbnXRdl3/+85/MnDmTnTt3snDhQgYNGsTBgwdz9Joikj22ZXismc3Mrjaf7QnQdLrL+gPa/C4ikh1ZBqzt27fToEEDHnvsMVzXJTExEdd1efTRR2nQoAHbt2/P1gvFxcWxa9cuXNcFMsJVeno68fHxme5bu3Ytu3fv5sILLwSgWbNmVKpUiXXr1mX3ZxORc3BJVYs1lzqUcKDlTJd3tqqVg4jI2coyYN1xxx1ER0ezbds2Fi9ezPvvv8/ixYv57rvvKF26NHfeeWe2XqhcuXI0atSIiRMnAhmNTBMSEk7Zf/VHENuyZQsA3333Hdu3bycxMTGbP5qInKsaUYZlfRwur2EYtMTjhqUex9TKQUTkL2W5B2vx4sVMmDDhlBmmuLg4HnzwQa6//vpsv9j48eMZPHgwY8aMISoqirfffvvktZ49e/Lwww/TtGlTxo8fT//+/bEs62TfrcqVK2f79UTk3BVzDG+0s2ldPsCILz3W7A8wpbNN1VJq5SAikpUzforwjw3ppzzIcXL0KcOkpKRT2jL8Ye7cuSf//corr+TKK6/M9vOLSO4wxjC0tqHRedB/gUfj6S6TOtp0i8v9Vg4iIuEoy9+OF154IY888sgpm8sPHTrEY489Rps2bXK9OBHJX5qWtUi51KFFOUOPTzweWuPhn76VnohIoZblDNbTTz9N27ZtqVq1Kp06daJChQrs3buXhQsXEhkZmWl5T0QKj5iiho+62Tz6tc+Da3yW7wswsaPNeUW1ZCgi8ocsZ7Dq16/PN998w9ChQ9mzZw+LFi1iz549DBs2jHXr1lGvXr1Q1iki+YhlDKMb23zc3WbljwGaTHdZ/aM+ZSgi8ofTzmAdPXqUu+66i7/97W8nzxsUEfmzbnEWKZcaBiz0uHCWxwutYWhtc0rPOxGRwua0M1hFixblzTff5MiRI6GuR0TCTNVShqW9bYYkWQz/wuPaJR6HXe3LEpHCLcslwtatW7NixYpQ1iIiYaqIbXipjc07HWwmf5/R/f2L/2jJUEQKryw3uT/88MMMHDgQx3Ho0aMH5cqVO2XaX+cXich/+1sti0bnGYZ87tF2tsew2gEeb25RpoiWDEWkcDGB0x0ICFjW/09uZbWfwvO83KnqLKWkpNCkSRPWrFmT48OeDx48qKAYxjR++ZPnB3hlk88/VvkUd+BfrWwur37q3iyNX/jTGIY3jV/uyXIG64033tBGVRHJEdsy3FzPpm+Cxd+XeVy5yOPtrYaXLrSpFqXfKyJS8GUZsAYPHhzCMkSkIKpcwjCls8PsnT43f+lRb4rLA40tbm9gEWEpaIlIwZXlJvfq1auzbt26017bsGED1atXz7WiRKRg6V3VYuMAhxvqWIxa7dNkusvyvdoELyIFV5YBKzU1lWPHjp322uHDh0lPT8+1okSk4CkZYRjbymZVX4dIy9B6lsddXxfh5+Nq6SAiBU+mJcKjR49y+PBh/tj3/ssvv5xyFuHRo0eZMWMGlSpVCl2VIlJgNI41rOhjM26jz/+siuDjD12eb2WTXE0NSkWk4MgUsJ544gkefvhhIOOTg926dcvygQ8++GCuFiYiBZdtGf5e36ZT6V+4b2MpBiz06BVnePFCm6qlFLJEJPxlClh9+/YlISGBQCDAddddx3333UeNGjUyPSAyMpI6depwwQUXhLRQESl4KhcPMLOrw4xUnxFfedSd4vJIE4uR9S0cbYIXkTCWKWA1bNiQhg0bAhkzWL169SI2NjZPChORwqNvgkWnSob7V/vcucJn4nc+r7a1aVo2y22iIiL5Wpa/vQYNGqRwJSIhExVpeK61zYq+NgGgxUyPv3/l8as2wYtIGMoyYB05coRRo0aRmJhI8eLFsW070z+Ok2ULLRGRHGtW1mJVX4cnm1u8vsWnzhSXZWrpICJhJsuUdPPNNzNp0iQGDBjAtddeS2RkZCjrEpFCzLEMdzSw6V/N4urFHl3mesztDu0qaslQRMJDlgFr9uzZPPXUU9xyyy2hrEdE5KSqpQzzetj0me/R/WOPWd2gc2WFLBHJ/7L8TWXbNklJSaGsRUTkFCUiDLO72XSoZLh4nsfcNC0Xikj+l2XAuuGGG3j33XdDWYuIyGkVcwzTu9h0r2Lo+6nHzFSFLBHJ37JcIixRogRLly6lVatWdOnShdKlS2e6bozhtttuy/UCRUQAitiGDzvbXL3Io/8Cj/c6wmU1tFwoIvlTlgHrnnvuASAtLY0VK1accl0BS0RCLcIyTOpkM3iJx5WLPY77MLCWQpaI5D9ZBizf1xS8iOQ/jmV4u71NEcvjms8yQtZ1SQpZIpK/qJmViIQd2zK81s6miO0z5HOPY16AG+vaeV2WiMhJmf7aN2bMGPbs2ZPphs8//5zff/890/e+//57rrvuutyvTkQkC5YxvHihxa31LW760ufZb7y8LklE5KRMAev+++8nPT395Nee59GxY0e2bNmS6UE//vgjb7/9dmgqFBHJgjGGsS0t7m1ocftyn8fXKmSJSP6QaYkwEDj1zK/TfU9EJL8wxjCmmUURG/6xyueoBw80tjDG5HVpIlKIaQ+WiIQ9YwwPNrEpYsOoVT7HPBjTTCFLRPKOApaIFBj/uMCmqA23L8+YyRrbUiFLRPLGKQHrdL+M9AtKRMLFbednzGTd/GXGTNa4Cy0s/Q4TkRA7JWB17NgRy8rcU6Zt27aZvqceWSKSn91U16aIZRi61OO4H2B8GxvbUsgSkdDJFLAeeOCBvKpDRCSohtS2iLRh8BKPlP0uvatadK1saF7OEKGwJSK5TAFLRAqsv9WyKF8MXt/s88K3Pg+nQFQEdKpk6FrF0LWKRY0ohS0RCT5tcheRAq1rFYuuVSw8P8Ca/QHm7wow/4cAI7/ycQM+1UpB18oWXasYOlUylC6iwCUi504BS0QKBdvKWB5sXg7uawy/HA/w2Z4An+4KMP8Hn/GbwTLQomzG7FaXyoaW5Yz2bolIjihgiUihFBVpuKSq4ZKqADapvwb49IcA83f5PLfB56EUaFbWMLWzTVxJhSwRyR4dQS8iAiSUMgytbfFhZ4f9f3NY2NNm75EATaa7LN6tT06LSPYoYImI/IltGTpVtljd16FBjKHLXI+x6z0dHSYiZy1bASs1NZUFCxZw8ODB3KpHRCTfKFvM8EkPmzvOt7hjhc+Vizx+P6GQJSJ/LcuAdccdd3Drrbee/Hr69OkkJSXRtWtXatWqxZo1a0JSoIhIXnIsw4E8VjcAACAASURBVBMtbCZfZPNRWoCWM12++1khS0TOLMuANX36dJo2bXry61GjRtGzZ0/Wr19P8+bNue+++0JSoIhIfjCgusWKPg7HPGg6w2VOmvZliUjWsgxYe/bsIT4+HoDt27ezZcsW7rvvPurXr8/IkSNZvXp1yIoUEckP6sUYVl3q0L6i4eJ5Hg+t8fC1L0tETiPLgBUdHc2+ffsA+PTTT4mJiaFJkyYAREZGcuTIkdBUKCKSj0RHGqZ3sXmkicVDKT595nv8dEwhS0Qyy7IPVrt27Rg9ejR79+7l6aefpm/fvievbdmy5eTslohIYWMZw32NbRrHGq5e7NFshsv0Lg71Y9QvS0QyZDmD9eyzz1KhQgXuvfde4uPjeeyxx05ee/fdd2nbtm1IChQRya96xlusvtShmAMtZrp8sF37skQkQ5YzWJUrV2bRokWnvTZv3jyKFi2aa0WJiISLGlGGZZc4DF3qccUij1U/Bni8uYWjI3ZECrUc9cFyXZfIyMjcqklEJKyUiDC819FmbEuLf23w6TrX48cj2pclUpipD5aISBAYY7jtfJsFPW02HMo4YmfVj1oyFCms1AdLRCSIOlSySLnUoWJxQ5tZHhM2K2SJFEbqgyUiEmRVSho+720zKNFw/VKPG5Z6HPO0ZChSmKgPlohILihiG15t6/BqW5s3t/p0+Mjjh98VskQKC/XBEhHJRUNrWzSMgeQFHo2nu3x4kU27itn6fJGIhCH1wRIRyWXNy1msudShTmnDRXM8nt/gEdAROyIFmvpgiYiEQLlihgU9be5Z6fP3ZT4r9gV4ta1NiQj1yxIpiLIMWH84fPgwX3/9NQcPHiQmJobGjRsTFRUVitpERAoUxzI809KmWVnDkM89vjnkMq2zQ81ohSyRguaMGwEee+wxKlSoQLt27ejTpw9t27alfPnyjBkzJlT1iYgUOFfUsFjRx+GoC01nuHy0U60cRAqaLAPWc889x/33389VV13FokWL2LRpE4sXL+bqq69m9OjRPP/886GsU0SkQKkfY1h1qUP7iobe8z1Gr/bwfO3LEikoslwifPHFF7nrrrt44oknTn4vKSmJ9u3bExUVxbhx4xg5cmRIihQRKYiiIw3Tu9j8c63P/at9Vv0Y4L2ONjFFtWQoEu6ynMFKS0ujS5cup73WuXNn0tLScq0oEZHCwjKG/2lk83F3m5U/Bmg6w+Xr/ZrJEgl3WQasSpUq8cUXX5z22pdffkmlSpWy/WLbtm2jdevWJCYm0rx5czZu3Hja+44dO8aIESOoVasW9erVY+DAgdl+LRGRcNItLqOVQ5ki0HqWyztbtS9LJJxluUR4/fXX88ADD3Ds2DEuu+wyKlSowN69e5k8eTJPP/00Dz30ULZfbPjw4QwbNozBgwczZcoUhgwZwrJly065795778WyLLZu3Yoxhj179mT7tUREwk1CKcMXvR1u/tJj0BKPFT8GeLalRaStJUORcGMCWXS7CwQC3HnnnYwbNw7XdU9+33EcRo4cyVNPPZWtF9q3bx+JiYns378fx3EIBAJUrFiR5cuXk5CQcPK+33//ncqVK7Nr1y5Klix5xudMSUmhSZMmrFmzhsaNG2ernj/80X5CwpPGL7xp/E4vEAjw2uYAt3zl0STW8GFnm8ol8mfI0hiGN41f7jntDFYgEODgwYM88sgjjBo1ihUrVnDo0CFiYmJo3rw55513XrZfKD09nUqVKuE4GS9pjCE+Pp60tLRMAWv79u2cd955PProoyxYsIBixYrx4IMPctFFF2X53CNGjCA6Opp+/fqRnJycrboOHTqU7Z9F8g+NX3jT+GWtf3mo1t5i8PJiNJrqMaHFUS4s6+V1WafQGIY3jV/w/DmonjZgua5LhQoVmDFjBr169aJnz55BeXFjMv8N7HSTZydOnOD777+nbt26PP7446xbt47OnTuzceNGypYte9rnHTduXI5nsODUPxQJLxq/8Kbxy1qXGPi6coArFnpcurQ4Tza3uO1865TfpXlNYxjeNH6547Sb3CMiIqhcuTKeF7y/LcXFxbFr166Ty42BQID09PRTDo2uWrUqlmVx9dVXA9CwYUOqVavGt99+G7RaRETCRblihvk9be443+KOFT79F3js/FWfMhTJ77L8FOGIESN49tlnOXbsWFBeqFy5cjRq1IiJEycCMHXqVBISEjItDwLExsZy0UUXMW/ePAB27tzJjh07SEpKCkodIiLhxrEMT7SwmdLZ5ou9ARInu9y+zGP/UQUtkfwqy08R7ty5ky1bthAfH0+HDh0oV65cpmlpYwzPPfdctl5s/PjxDB48mDFjxhAVFcXbb7998lrPnj15+OGHadq0Ka+88grXXXcd99xzD7Zt8+qrr1KxYsUc/HgiIgVHcjWLblUMz37j89R6n9e3+NzVIGPZsKQOjRbJV7L8FGG1atXO/EBj+P7773OlqLOlTxGKxi+8afxybv/RAGO+9nlxo0/pInB/I4thtUPf0kFjGN40frknyxmsHTt2hLIOERHJhtiihrGtbG493+KBNR5/X+Yz9hufR5raXFnDYOWzjfAihU2We7BERCT/iy9peLO9w/p+Dg1iDAMXezSa5jI3zT/tJ7VFJDQyBazvv/+eGjVqMGfOnCwfMGfOHGrUqMHmzZtzvTgRETk79WIMM7o6fHmJTXSkodc8jw4feSzbqyN3RPJCpoD1zDPPUL16dXr16pXlA3r16kWtWrUYO3ZsrhcnIiLZ07q8xZKLbeZ0s/npeIDWszz6znfZeEizWSKhlClgzZkzh0GDBv3lg/72t7/x6aef5lpRIiKSc8YYesZbfN3PYWJHm/UHA5w/1eXaJS5pvyloiYRCpoC1Z8+eU/pSnU7VqlXZvXt3btUkIiJBYBnD1TUtNg9weK6Vxdz0ALU+UA8tkVDIFLBKlSrFvn37/vJB+/bto1SpUrlWlIiIBE+kbRhRz2b75Q73NbJ4fYtPjX+7PJri8dsJBS2R3JApYDVr1oz333//Lx/0/vvv06xZs1wrSkREgq9khOH+xhlB67oki0e+9qn5gcuEzT6er6AlEkyZAtYtt9zC1KlTuf/++097DqHneYwePZpp06YxcuTIkBUpIiLBU7aY4dlWNlsvc7iokuH6pR5NZ7h8tlufOBQJlkyNRnv27MmoUaN47LHHmDBhAp07dyYuLg5jDGlpaSxYsIC9e/cyatQoevTokVc1i4hIEFQtZXivk8OIej63LffpOMejX4LPky1sakSpUanIuTilk/ujjz5KmzZtePrpp5kyZQpHjx4FoGjRorRp04Y333yTbt26hbxQERHJHa3KW3x1ieHf2wPcs9Kj7ocuf69v8T+NLKIjFbREcuK0R+V0796d7t2743keBw4cIBAIEBsbi23boa5PRERCwDKGq2oa+iYYnl7v88Q6n7e2+jzS1OL6JAvbUtASyY4zHpVj2zblypWjfPnyClciIoVAcccwunHG/qwecYYbvvBpNN1l4Q/anyWSHTqLUERETlG5hOHtDg4r+9qUijB0nuvRZ77Ltp/1aUORs6GAJSIiWWpW1uKL3jb/7mSz7kCAelMyGpUeOqagJXImClgiInJGxhgur2GxaYDDg40tXt3sU+sDl5c2erhaORQ5LQUsERE5K8Ucw6hGNtsud7ikqmHElz7tFhTn/e98jrqa0RL5bwpYIiKSLRWLG95o77D6UofyRQNctdij8iSX25Z5fHtQQUsEFLBERCSHGscaprc7wqYBGUfvvPedT/2pLq1muryxxdc5h1KoKWCJiMg5qV3a8FQLm11XOUzpbBMdCdd/7lHxPZdhS11W7vMJBBS2pHA5baNRERGR7Iq0DcnVDMnVLHb+GuDNrT5vbPF5bbNHgxi4Psni6poWMUXVtFQKPs1giYhI0FUtZXiwic2OKxzmdrepGWW4fblPpUkuAxdnHCytWS0pyDSDJSIiuca2DD3iDD3iLPYeDvDONp/Xt/i8951HzaiMWa1BiRYVimtWSwoWzWCJiEhIlC9uuKuhzeYBDksutmlVzvBgik+VSS6XzneZm+bj+ZrVkoJBAUtERELKGEO7ihbvdHTYfbXDc60sdvwaoNc8jwZTXb5RqwcpABSwREQkz5QpYri5ns3X/RxW9LGxLWg+w+X1zdqjJeFNAUtERPKcMYbm5SxW9HG4ppbF0KUeAxd7/HpcIUvCkwKWiIjkG8Ucw/i2Nu93spmVFqDpDJe1BxSyJPwoYImISL5zRQ2LlEsdSjjQcqbLyxs9LRlKWFHAEhGRfKlWtOGrSxyuT7K46UufKxZ5/KwlQwkTClgiIpJvFXUM4y60+fAim0/SAzSe5rLmR4Usyf8UsEREJN/rX93i634OMUUMrWe5vLBBS4aSvylgiYhIWKgeZfjiEpsb61qMXOaTvMDj0DGFLMmfFLBERCRsFLEN/2plM72LzeLdGUuGK/f5eV2WyCkUsEREJOz0TbBY28+hfHHDhbM8xq7XkqHkLwpYIiISlqqWMnx+sc3f61vcscKnz3yPg0cVsiR/UMASEZGwFWkbnm5pM7urzZd7AzSY5jJrp5YMJe8pYImISNi7uGrGkmHDGEOf+R4DFrjsOazZLMk7ClgiIlIgxJU0fNTNZlJHmyV7AtT5UIdGS95RwBIRkQLDGMOVNS02DXC4NMEwdKlHxzkeW39SyJLQUsASEZEC57yihjfbOyzoabPr94y9WY997XHcU9CS0FDAEhGRAuuiyhbrkx1urW/xwBqfpjNcVqhvloSAApaIiBRoxR3D481tVvV1iLQMrWZ6/P0rj191cLTkIgUsEREpFBrFGpb3sXm6hcXrW3zqTXGZk6bZLMkdClgiIlJoOJbh9gY2G5Id6pQ2XDzP48pFLnvV0kGCTAFLREQKnWpRhk962LzbwebTXQHqTHF5c4taOkjwKGCJiEihZIxhYC2LzZc5XBxnuO5zjy5zPbb/opAl504BS0RECrXYooZ3OjrM62Gz/ZcA9ae4PLHW44SvoCU5p4AlIiICdK1isaG/w811LUat9mk+w2X9AYUsyRkFLBERkf9TIiLj8OiVfRy8ADSd4fLPtR6uZrMkmxSwRERE/qRJWcOqvg53nG9x32qftrN13I5kjwKWiIjIaRSxDf9sbrO0t83+owEumObywgYPX580lLOggCUiInIGrctbrO3nMCTJYuQyny5zPXb+qpAlZ6aAJSIi8hdKRBheuNBmQU+bbT8HOH+q+mbJmSlgiYiInKWLKlt8098huVpG36w+8z3+oy7wchoKWCIiItkQHWl4s73DzK42K37M6Jv14fc601AyU8ASERHJgUuqWnzb36FjJcNlCz2uWuRy8KhmsySDApaIiEgOxRY1TL7I5r2ONh+nB6g/1eXjdM1miQKWiIjIOTHGcFXNjC7wDWMMPT/xGLbU5dfjms0qzBSwREREgqByCcPc7jbj29hM+i5Aw2kun+/RbFZhpYAlIiISJMYYhtWxWJ/sUKWEocNHHtcsdlnzo2azChsFLBERkSCrHmVY3Mvm+dYWS/4ToOkMlwtnufx7u88JnWtYKIQsYG3bto3WrVuTmJhI8+bN2bhx4xnvf+ihhzDGsGHDhhBVKCIiEjy2ZRhRz2b75Q7TOtsUseDKRR4J77s8muKx74iCVkEWsoA1fPhwhg0bxtatW7n77rsZMmRIlvempKSwfPly4uPjQ1WeiIhIrnAsw6XVLBZd7LA+2eHieIsxa33iJrkM+sxl9Y/ap1UQmUAI+vzv27ePxMRE9u/fj+M4BAIBKlasyPLly0lISMh077Fjx+jQoQOTJk2iY8eOfPTRR9SvX/+0z5uSkkKTJk1o1aoV0dHR9OvXj+Tk5GzVdujQIcqUKZPTH03ymMYvvGn8wp/GMGcOHYf3UiOYsD2StMMWTWM8htU8Tu/KLpEh3Lyj8QuemJiYTF87oXjR9PR0KlWqhONkvJwxhvj4eNLS0k4JWKNHj2bgwIFUq1btrJ9/3LhxNG7cOMf1/fkPRcKLxi+8afzCn8Yw+2KA0RXgf5oH+CgtwAvfGoattKlQDG6sazGstkWF4iY0tWj8ckXIcrIxmf9DOd3E2bJly1i1ahU33XRTqMoSERHJM7Zl6JNgsaCXw4Zkh74JFk+s84l/32XgYpeV+7R8GK5CErDi4uLYtWsXrusCGeEqPT39lD1WS5YsYfPmzVSrVo2EhAR27dpFt27d+Pjjj0NRpoiISJ6pF2N4uY3NrqscHm9m8dXeAC1merSY4TJ9h4JWuAlJwCpXrhyNGjVi4sSJAEydOpWEhIRTlgfvvfdedu/eTWpqKqmpqVSpUoV58+bRo0ePUJQpIiKS58oUMdzewGbbZQ6zutqUioB+CzzuWO7hqcVD2AjZEuH48eMZP348iYmJPP7440yYMOHktZ49e7J69epQlSIiIpLv2Zahd1WLT3vaPN/K4l8bfC6e5/GzjuAJCyHZ5A6QlJTEsmXLTntt7ty5p/1+ampqLlYkIiKS/xljuKW+TVJpw2ULPVrOdJnd1aFmdGg2wUvOqJO7iIhIGOhaxWJFHwc/AM1nuiz6Qfuy8jMFLBERkTCRVNqwvI9D01hD1489Xt7o5XVJkgUFLBERkTBSpohhbnebm+ta3PSlz81fejrfMB8K2R4sERERCQ7HMjzX2qZeGcPNX3ps/inAhxfZxBTVvqz8QjNYIiIiYWpYnYxPGa49EKDFTJdNhzSTlV8oYImIiISxDpUsVvV1iLSg5UyXT9K1+T0/UMASEREJc9WjDMv6OLStYOg1z+Nf33inPZJOQkcBS0REpACIijTM7Gpzx/kWty33GbrU47inkJVXtMldRESkgLAtw5MtMja/D1vqsfVnj6mdbcoW0+b3UNMMloiISAEzKNFi8cU2W34O0GyGyzcHNZMVagpYIiIiBVDr8hmb30tHQutZLrN2avN7KClgiYiIFFDxJQ1fXOLQpbKh73yPYUtdPk73OeJqRiu3aQ+WiIhIAVYywjCls82T63zGb/J5bbNHURs6VjK0j4lgQO0A1aO0RyvYFLBEREQKOMsY7r3A5p6GFpt/go/TfeamB7h/fRHuXeeSFA094yx6xBnaVTQUsRW4zpUCloiISCFhjKFOGahTxub2BrBz70G+PlKauek+k3f4PLsBSjhwUWVDzzhDjziL+JIKWzmhgCUiIlJIlYqAvuUt+iZYBAIBNhyCuWkZs1s3f+njBXzql4EecRY94wwXVjBEWApcZ0MBS0RERDDGcH4MnB9jc88F8NOxAJ/+EGBuus8723yeWg9REdCliuGSeIuraxpsha0sKWCJiIjIKUoXMQyobhhQ3cIPBFh74P9ntwYv8Xhrq2FiR5tKJRSyTkdtGkREROSMLGNoHGu4r7HNV30cFvXKaGLacFpG2wc5lQKWiIiIZEuHShbrkh1alDX0/MTjzuU69/DPFLBEREQk22KLGmZ3s3mmhcXz3/q0me3x/S8KWX9QwBIREZEcMcZwewObLy+xOXA0wAXTXD7YriVDUMASERGRc9SsrMXX/Rx6xRuuWOQx9HOXw4X8OB4FLBERETlnUZGGSR1tXm9r8953AZpNd9lwsPCGLAUsERERCQpjDENqW6y+1MEy0GyGy6ubfAKBwhe0FLBEREQkqOqWMazs6zColsXwLzwuX+jx07HCFbIUsERERCToijmGV9raTL7IZt6uAI2muazYV3g2wCtgiYiISK4ZUN1ibT+HcsUMbWZ5PLXOwy8ES4YKWCIiIpKrqkUZlva2ue18i7tX+vT6xGPfkYIdsnQWoYiIiOS6SNvwZAubTpUM13zm0XCqy+jGFmWLGkpG8H//GEo6nPy6uJNxTE84UsASERGRkOkeZ7Eu2TD4M4+bvvzrPVkl/itwZYQvc8rXjWMNV9QwFHfyTxhTwBIREZGQqljcMK+nwwk/wO8n4LcT8JsLv50IZPz7ya///3u//umeX4/DnsPwy3GflzbCXStgaG2Lm+paxJfM+6ClgCUiIiJ5IsIylC4CpYv88Z2cBaPtvwR48Vuflzf6PLXe59IEw8h6Fm0rGEweLTFqk7uIiIiEtRpRhrGtbH642uGF1hbfHgrQ/iOPC6a5TNjscyQPju1RwBIREZECoWSE4aa6Nhv7O8zvYRNf0jB0qUeVSS7/WOmR9lvogpYCloiIiBQoxhi6VLGY3c1h2+UZHeVf2uhT/d8u/Re4fL4n94/vUcASERGRAuu/lw+f/6/lw0bTXN7YknvLhwpYIiIiUuD9sXz4bX+HeT1s4koarv/cI+7/lg/Tg7x8qE8RioiISKFhGUPXKoauVSy++znAixt9Xtros+v3AO92DF4sUsASERGRQqlmtOHZVjYPN7H4zQ3ucytgiYiISKFWKtJQKjK4z6k9WCIiIiJBpoAlIiIiEmQKWCIiIiJBpoAlIiIiEmSFPmBNnTo1r0uQc6DxC28av/CnMQxvGr/cU+gD1rRp0/K6BDkHGr/wpvELfxrD8Kbxyz1h3abhyJEjAGzatCnHz/Hzzz+TkpISrJIkxDR+4U3jF/40huFN4xdctWvXpnjx4gCYQG6fdpiL3nvvPQYOHJjXZYiIiIiwZs0aGjduDIR5wNq/fz/z5s0jISGBYsWK5XU5IiIiUogVmBksERERkfyo0G9yFxEREQk2BSwRERGRIFPAEhEREQkyBSwRERGRIAvrPljBcuzYMYYPH05UVBSe5/Hiiy+e1TXJH840RqmpqfTo0YP27dtToUIFHnzwwbwrVE7r0KFD3HnnncyfP5/09PRM1/T+Cw9nGkO9B/O/JUuW8M4773D06FHKlCnDuHHjTl7TezDnCtwM1siRI0lISMAYw4YNGzJd27ZtG61btyYxMZHmzZuzceNGIKOTbfv27Xn++ecpU6YMy5YtO/mYM12T4Av2+AFER0dz9OhRqlWrFrKfo7DKyfiVKVOGCRMmkJSUdMrz6f0XesEeQ9B7MJRyMn7t27dnwoQJvPfee6SlpfHrr7+efIzegzlX4AJW//79+eKLL6hateop14YPH86wYcPYunUrd999N0OGDAFg586dJCQkAFC9enV27tx58jFnuibBF+zxq1q1KsuXL+fNN99k1qxZpKamhuLHKLRyMn5novdf6AV7DPUeDK1zGb+5c+dSp04dSpUqdfJ7eg/mXIELWO3ataNKlSqnfH/fvn2kpKSc7PyenJzMjh07SE1NJS4u7uR/NKmpqcTHx5983JmuSfAFe/yMMSf/t3z58pn+ZibBl5PxOxO9/0Iv2GOo92Bo5XT8XnvtNVauXMkTTzyR6XF6D+ZcodmDlZ6eTqVKlXCcjB/ZGEN8fDxpaWkkJydzww03sH79eo4cOULr1q3ZsmUL48ePZ8yYMadck9DL6fj17t2bd955B4DSpUtz/vnn5+WPUWidafwSEhK44YYb2Lx5MzfccAN33XUXruvq/ZfP5HQM9R7MH840fmvWrOHhhx+mV69e3HDDDTzyyCMcPHhQ78FzVGgCFvz/36T+8EcT+6JFi/LWW29lupaUlMTYsWMBTrkmeSOn49exY8eQ1CdnltX4Abzyyiun3K/3X/6T0zHUezB/yGr8kpOTSU5OznStbNmyeg+eowK3RJiVuLg4du3aheu6QMZ/WOnp6ZruDBMav/Cm8Qt/GsPwpvELvUITsMqVK0ejRo2YOHEiAFOnTiUhIeHk5j3J3zR+4U3jF/40huFN45cHAgXMTTfdFKhcuXLAtu1A+fLlAzVq1Dh5bfPmzYGWLVsGatWqFWjSpElgw4YNeVipnI7GL7xp/MKfxjC8afzyDxMI/NciuoiIiIics0KzRCgiIiISKgpYIiIiIkGmgCUiIiISZApYIiIiIkGmgCUiIiISZApYIiIiIkGmgCUiIiISZApYIiIiIkGmgCUikg0PPvggJUuWzOsyRCSfU8ASERERCTIFLBEREZEgU8ASkXxv2bJldOrUiRIlShAdHc1VV13Fvn37AEhNTcUYw9tvv82QIUOIjo4mJiaG22+/Hdd1Mz3Phg0b6N69OyVLliQqKoo+ffrw3XffZbrH933Gjh1LnTp1KFKkCBUqVGDAgAH8/PPPme5bv349bdq0oXjx4tSvX5958+Zluj5r1iyaNm1KyZIlKV26NE2bNmXu3Lm58KcjIvmRApaI5GvLli2jQ4cOREdH88EHH/Dqq6+yatUqLrnkkkz3jRo1Ct/3mTx5MnfddRcvvPAC991338nr6enptG3blr179/L222/z+uuvs3XrVtq2bcuPP/548r5bbrmFu+++m4svvpjZs2fz4osvUqpUKX777beT95w4cYKBAwcyePBgpk+fTmxsLMnJyRw4cACA7du3079/f+rVq8f06dP54IMPuOyyy/63nfsHabuJ4zj+/qkRmqLUWCWhihVEpzq5NNSKsYguUsU/qdAhuIguRpxaIYJYECkuLtLBoW6CDhooiFJqdRFcFESKKK2oFYzoEEQSr0PpD1Lp0z5PU0wfPi/4Qe5y9833d9OXuyOcnJz84dUSkZRhRERS2MOHD43X6zWXl5d238bGhrEsy4TDYbOzs2MAU1lZmTCvv7/fOJ1OE4lEjDHGBINB43Q6zdHRkT1md3fXOBwOEwqFjDHGbG1tGcuyzIsXL36YTygUMoAJh8N234cPHwxgXr9+bYwxZmpqygDm7Ozst99fRP5O2sESkZQVjUZZXl6mpaWFeDxOLBYjFotRVlaGx+NhdXXVHtvY2Jgwt6mpiWg0yvr6OgBLS0v4fD7y8vLsMUVFRXi9XpaWlgBYXFzEGENHR8c/5pWWlsajR4/sdklJCZmZmezt7QFQXl5Oeno67e3tzM7OXjleFJH/PxVYIpKyTk5OiMfjBINBHA5HwrO/v8+nT5/ssfn5+Qlzv7UPDg7sWG63+8pvuN1uIpEIAMfHx2RkZFyJ9b0bN26QmZmZSqhjvwAAAj1JREFU0OdwODg/PwegtLSUubk5Tk9PaWxsJC8vj4aGBj5+/PgvV0BE/lYZ152AiMiP3Lp1C8uyePbsGY8fP77y/e3bt+3P3y69f9/2eDwAuFwuPn/+fCXG4eEhLpcLgNzcXGKxGEdHRz8tsn6mrq6Ouro6zs7OePPmDcFgkEAgwMLCwm/FFZG/g3awRCRl3bx5k/v377O5uUlFRcWV5+7du/bYmZmZhLnT09M4nU7u3bsHwIMHD1hYWLAvosPXi+8rKytUVlYC4PP5sCyLiYmJpL1DdnY2ra2t+P1+Njc3kxZXRFKbdrBEJKWNjIzg8/loa2vD7/eTk5PD3t4e8/PzBAIBu8ja3t4mEAjg9/tZW1tjeHiYnp4ecnJyAAgGg0xMTFBbW8vz58+Jx+OEQiFcLhfd3d3A16O9zs5O+vv7iUQi1NTUEI1GCYfDDAwMcOfOnV/KeXx8nJWVFerr6/F4POzs7DA5OUltbe0fWSMRST0qsEQkpXm9Xt6/f08oFCIQCHBxcUFBQQE1NTWUlJTY/3U1NDTE27dvaWlpIT09na6uLoaGhuw4hYWFvHv3jr6+Pp4+fUpaWhrV1dW8fPky4eL72NgYxcXFvHr1itHRUXJzc6mqqiIrK+uXcy4vL2d2dpbe3l6Oj49xu908efKEwcHB5C2MiKQ0yxhjrjsJEZH/and3l+LiYqampmhubr7udEREAN3BEhEREUk6FVgiIiIiSaYjQhEREZEk0w6WiIiISJKpwBIRERFJsi/ybu88QpOxtAAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAFyCAYAAAApuaQRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3xUVd7H8c+59yb0BGLoJISW0JFepEtHBAlYWUERsCBr1+WxF9aKq2JBxYq4Ih1BQYqISo+ASBUJCcKCFCv13jvPH3nk2QhBEiaTTPJ9v16+1uTemfmF42S/nHPmd0wgEAggIiIiIkFj5XUBIiIiIgVNWAesw4cPk5KSwuHDh/O6FBEREZGTwjpgbd68maZNm7J58+YcP8fPP/8cxIok1DR+4U3jF/40huFN45d7wjpgBYPneXldgpwDjV940/iFP41heNP45Z5CH7BEREREgk0BS0RERCTIFLBEREREgszJ6wJEREQk/KSlpbF///68LiNPxMbGEh8ff8Z7FLBEREQkW9LS0qhTp06hbZNUvHhxNm3adMaQpYAlIiIi2bJ//34OHz7MxIkTqVOnTl6XE1KbNm1i0KBB7N+/XwErK4FAgF2HDTExeV2JiIhI+KlTpw5NmjTJ6zLypUK9yX3qjgDN55Vg9CqP30/oSEYREREJjkIdsHrFG0YlHmfsNz5JH7r8e7uPzr4WERGRc1WoA1Zxx/CPesfZOMChWazhikUeHT/yWH9AIUtERERyrlAHrD9UjzLM6ObwSQ+bvUcCNJ7uMvJLj4NHFbREREQk+xSw/kv3OIv1yQ5PtLB4e5tP4mSXVzf5eL6CloiIiJw9Baw/ibQNdzS02XqpQ+94w4gvPFrMdPlqr5/XpYmIiEiYUMDKQsXihrc7Onx5sQ3ABbM8rl7ssuewZrNERETCVWpqKh07diQ6OppmzZrl2usoYP2FNuUtVvZ1GN/WZm56gKTJLk+v9zjuKWiJiIiEm6ioKB599FEmTZqUq6+jgHUWbMswvI7FtsscBteyuHulT8OpLvPStWwoIiKSHz311FOMGDHi5Nc//fQTsbGxALRt25YSJUrk6usX6k7u2VWmiOGFC2yG1ba4+SuPHp949K3qM7aVTfUok9fliYiI5Ct7DgfYkwvHFVYsnrGV50yGDRtGUlISTz75JNHR0UyYMIG+ffsSE6LjWxSwcqDheYbPLrL54PsAd6zwqDvF5a6GFvecb1HcUdASEREBGL/J56GU4K/2PNDE4sGm9hnvKV26NMnJybz11luMGjWKl19+mQ8//DDotWRFASuHjDFcXsPQJ94wZq3PE+t83trq80wrmwHVDMYoaImISOE2oo7FxVWDvxupYvGzu2/UqFH069ePGjVqUL58eRo3bhz0WrKigHWOSkQYHmtuc02ixa3LPS5d6NGpouGFNjb1YhSyRESk8KpY3Jx1GMoNtWvXJiEhgRtuuIEnn3wypK+tTe5BUjPaMLu7w5zuNrt+D9Bomsstyzx+OqZPG4qIiOSVYcOG4bouAwYMAODYsWNUqVKFgQMHsn79eqpUqcI//vGPoL+uZrCCrFe8xYWVDf/6xueRr30mfefzz+Y21yQZLC0bioiIhNTChQu58cYbiYiIAKBIkSLs2rUr119XM1i5oIhtuPt8my2XOnStbLhuqUermR4r9qmtg4iISCjs3r2b2rVrs3btWm655ZaQv74CVi6qXMLwXmeHpX1sTvgBWs30uGaJy151gxcREclVlSpVYvPmzXz11VeUKlUq5K+vgBUCbStYrO7n8NIFFrN2Bkic7PLsNx4ndIi0iIhIgaSAFSK2ZbihbsYh0lfVtLhjhU+jqS4LftCyoYiISEGjgBVi5xU1vNTWZnU/h/OKGLrO9Uj+1CX1V81miYiIFBQKWHmkcazh8z4273WyWb4vQJ0PXR5a43HEVdASEREJdwpYecgYw5U1LTYPdPh7fYvH1vrU+dBl2g6fQEBBS0REJFwpYOUDpSINj7ew2ZDsULeMIXmBR7ePPTYdUsgSEREJR2o0mo8kljbM6W4zJy3ALcs9Gk51GVXf4v4mFtGRalIqIiL5y6ZNm/K6hJA7259ZASufMcZwUVVDl8qGsd/4PLbW573vfB5vYXN1LXWDFxGRvBcbG0vx4sUZNGhQXpeSJ4oXL05sbOwZ71HAyqeKOobRjW3+VsvizhUe1yzxGL/J8EIbi2ZltbIrIiJ5Jz4+nk2bNrF///68LiVPxMbGEh8ff8Z7FLDyubiShn9f6HBDXZ+bv/JoMcPj2iSfMc1tyhXTbJaIiOSN+Pj4vwwZhZmmQsJEh4oWKZc4PN/GYuqOjG7wz2/wcNUNXkREJN9RwAojjmUYWS+jG/xl1S1uWebTeJrL4t3qBi8iIpKfKGCFobLFDOPb2azq51AqwtB5jselC1zSftNsloiISH6ggBXGmpY1fHGxzTsdbZb+J0DtyS6PpngcVTd4ERGRPKWAFeYsY/hbLYstlzrcVM/ioRSfulNcZqaqG7yIiEheUcAqIKIiDU+1tPlmgEOtKEO/Tz16fuKx5SeFLBERkVBTwCpgapc2fNLTZkZXm60/B2gw1eWuFR6/HlfQEhERCRUFrALIGEPfBItvBzjc19hi3Lc+iZNd3t2mZUMREZFQUMAqwIo5hvua2Gwa6NCuguHqzzzazvZI2a+QJSIikpsUsAqBqqUMk7s4LOxl8/PxAM2mu1y/1GP/UQUtERGR3KCAVYh0rmzxdX+HZ1tb/Pv7jGXDF79VN3gREZFgU8AqZCIsw9/rZ3SD759guPkrn6bTXT7fo27wIiIiwaKAVUiVK2Z4vb3Din42RW1Dh488rlzksvewZrNERETOlQJWIde8rMWyvjZvdrD59IcAdaa4vLNVnzYUERE5FwpYgmUMQxItNg106BVnGLwko0npzl8VskRERHJCAUtOii1qmNjJ4aPuNt8eClBvisu4bz18zWaJiIhkiwKWnKJ3fEaT0r/Vsrj5K5/2sz0268gdERGRs6aAJacVFWl4ua3NZxfZ7DsSoNFUlzFfe5xQSwcREZG/pIAlZ9ShosW6ZIdbG1jcv8an+XRXneBFRET+QkgD1qhRo0hISMAYw4YNG7K8b9u2bbRp04bExERatGjBxo0bQ1il/Fkxx/B4C5sVfR0AWsxwuWelxxFXQUtEROR0QhqwBgwYwBdffEHVqlXPeN+IESMYPnw4W7du5a677mLo0KEhqlDOpGlZw6pLHB5uavHsNz7nT3NZqgalIiIipwhpwGrfvj1VqlQ54z379u0jJSWFQYMGAZCcnMyOHTtITU0NQYXyVyIsw+jGNmv7O8QWNbT/yOPGLzx+Oa7ZLBERkT84eV3An6Wnp1OpUiUcJ6M0Ywzx8fGkpaWRkJBw2seMHDmS6Oho+vfvT3JycrZe79ChQ+dacqFUHph5AUzYHsEjG4owM9VlbOOjdK3ohbQOjV940/iFP41heNP4BU9MTEymr/NdwIKMUPXf/qqr+Lhx42jSpEmOX+/Pfyhy9u4+Dy6vE2DEFx6Xf1Wcq2oa/tXaJrao+esHB4nGL7xp/MKfxjC8afxyR777FGFcXBy7du3CdV0gI1ylp6cTHx+fx5VJVqqWMnzcw+atDjZz0wPU/dDlg+06bkdERAqvfBewypUrR+PGjZk4cSIAU6dOJSEhIcvlQckfjDEMTrTYOMChQ0XD5Ys8+n3q8cPvClkiIlL4hDRg3XTTTVSpUoVdu3bRpUsXatasefJar169WL16NQDjx49n/PjxJCYm8vjjjzNhwoRQlinnoEJxw4ddHKZ1sVm5L2M267XNms0SEZHCxQTC+P/5UlJSaNq0KWvWrMnxHqyDBw9q/TmXHDoW4M4VHhO2BOhU0fBqO5ua0cHdm6XxC28av/CnMQxvGr/ck++WCKXgKFPE8Hp7h0972aT+FqDhVJdn1nu4Om5HREQKOAUsyXVdKlt8k+wwoo7FnSt82szy+OagQpaIiBRcClgSEiUiDM+2tvnqYpvfTwRoMs3lgTUexzwFLRERKXgUsCSkWpW3SOnvMLqxxZivfZpMc1m+V8ftiIhIwaKAJSFXxDY81NQmpb9DiQhDm1kety7z+P2EZrNERKRgUMCSPNMgxrDsYpunWlqM3+RTf4rLgh80myUiIuFPAUvylG0Zbm9osz7ZIaGUoetcj6FLXA4d02yWiIiELwUsyRdqRhsW9bZ5tZ3NlB0ZDUqn79BsloiIhCcFLMk3jDEMq22xcaBDi3KG/gs8Bi5w+c9hzWaJiEh4UcCSfKdyCcOMrjYfdLZZsidA3Skub2/VcTsiIhI+FLAkXzLGcGkNi00DHXrHGYYs8eg0x2PlPi0biohI/qeAJfnaeUUN73Zy+KSHzYGjAVrO9BiwwGXLT5rNEhGR/EsBS8JC9ziLtf0d3u5gs+rHAPWmuIxY6rHnSHAPjxYREQkGBSwJG7ZluDrRYstAh6daWkzZ4dN8XglGr/L4SW0dREQkH1HAkrBT1DHc2sDm+8sdbqh1nH9941PjA5dn1nscdRW0REQk7ylgSdiKjjT8T73jbL/c4dLqFnev9Emc7PLWVh/PV9ASEZG8o4AlYa9iccPLbW02DnBoVc5wzRKPRtNcZu9UawcREckbClhSYCSWNkzu4rCyn025ooaL53u0m+3x5X/U2kFEREJLAUsKnOZlLRb2tvmkh83vboC2sz36znf59qBms0REJDQUsKRAMsbQPc5izSUOkzrZfHMwQMNpLtcucUn/TUFLRERylwKWFGiWMVxR02LzQId/tbL4KC1Arckud67wOHhUQUtERHKHApYUCpG24eb6Ntsvc7inkcUrm3yqf+Dy+FqPw2rtICIiQaaAJYVKqUjDg00zgtbVtSzuX+NT6wOX1zb7uGrtICIiQaKAJYVSuWKG59vYbBro0KGiYfhSj/pTXKbtUGsHERE5dwpYUqjViDJM6uyQcolDQilD8gKP1rM8luxRawcREck5BSwRoHGs4ZOeDgt72Xg+dPzIo/cnau0gIiI5o4Al8l86V7ZY2c/mg842m3/KaO0wYqnHfw4raImIyNlTwBL5E2MMl9aw2DTQ4ZmWFlN2+NT8wOWRFI/fTyhoiYjIX1PAEslCpG24pYHNd5c53FDX4tGvMw6TfmOLDpMWEZEzU8AS+QtlihieammzeaBD+4qGoZ97NJ7uMn+XNsKLiMjpKWCJnKVqUYb3Ozss72sTHWHo/rFHj49d1h/QbJaIiGSmgCWSTS3LWXzex2Z6V5vvfwlw/jSXoUtcdv+uoCUiIhkUsERywBhDvwSLbwc6PN/GYubOjDMOH1jj8Zs2wouIFHoKWCLnIMIyjKxns/1yh5vrWTyxLuMThzp6R0SkcFPAEgmC6EjD4y1stgx06FI54+idRlNd5qbp6B0RkcJIAUskiKqWMkzs5LCqn03ZYobe8zy6zvVYq43wIiKFigKWSC5oVtZicW+bWd1sdv0eoMk0l8GfuaT/pqAlIlIYKGCJ5BJjDH2qWnwzwOHFCyw+Tg+QONnlmfUevpYNRUQKNAUskVwWYRluqJvREf7GuhZ3rPDp/YnHviMKWSIiBZUClkiIREUanmll80kPm5QDARpNdVn4g7rBi4gURApYIiHWPc5iXX+HBjGGrnM9Rq/yOKGWDiIiBYoClkgeqFDc8ElPm8dbWDy1zqf9bI/UXxWyREQKCgUskTxiGcNdjWyW9rH5z+GMI3c+/F5LhiIiBYEClkgea1Xe4uv+Dt2rGC5d6DFiqcdhV7NZIiLhTAFLJB8oXcTw7842r7WzeXebT4sZLhsOKmSJiIQrBSyRfMIYw3W1LVZf4mCA5jNcxm/ydNSOiEgYUsASyWfqljGs7OdwTaLF9V/4DFzoceiYQpaISDjJMmA1a9aMV155hZ9//jmU9YgIUMwxvNTWZmoXm4U/ZGyA/2qvNsCLiISLLANWYmIit912G5UqVeLqq69myZIloaxLRID+1SzW9neIK2FoP9tjzNcennpmiYjke1kGrEmTJrFnzx6eeuopNm3aRKdOnahZsyb//Oc/2b17dyhrFCnUqpYyfHaRzT/Ot7h3tU+3jz12/66QJSKSn51xD1Z0dDQ33ngjq1atYu3atVx00UWMHTuWqlWr0qdPH2bMmIHva9lCJLc5luGRZjYLe9ts/ilAo2kuT6/3+OW4gpaISH501pvcq1SpQrVq1Shfvjye57Ft2zaSk5OpVasWy5cvz80aReT/dKpksS7ZoU+8YfQqn7hJLnet8PhBM1oiIvnKXwasefPmcdlll1G5cmUee+wxunXrxrfffsvmzZvZunUrtWrV4tprrw1FrSICxBY1vNHBYcflDtfXsRi/ySfhfZfBn7l8o95ZIiL5QpYB6/777yc+Pp5evXqxf/9+3nzzTXbt2sXYsWOpU6cOADVq1OCBBx5gy5YtIStYRDJULmF4oqVN+pUOT7SwWLQ7QMOpLj0/dln0g6/+WSIieSjLgPXaa69x1VVXsXXrVhYuXMjll19OZGTkKfclJSXxxhtv5GqRIpK1qEjDbQ1tvr/c4d2ONrsPB7hwrkezGS7vf+fj6lOHIiIh52R1IT09HcfJ8vJJMTExDB48OKhFiUj2RViGQbUMV9U0fPpDgKfW+1y52OMfq+DWBhZDkyxKRpi8LlNEpFDIMkH9Ea62bt3KypUr2bNnDxUrVqR58+YkJSWFrEARyR5jDN2qGLpVsVh7IMDT6z1uX+7z4BqfG+pajKpnUaG4gpaISG7KMmD99ttvDB8+nMmTJ+P7PhEREZw4cQLLshg4cCCvvfYaJUuWDGWtIpJN559nmNjJYUzzAM9t8HnhW59n1vs82TIjaBmjoCUikhuy3IN1880389FHH/Hqq69y8OBBjh07xsGDBxk/fjxz5szh5ptvDmWdInIO4ksanmllk36Fww11LW5Z5nPFIo/fTmh/lohIbsgyYE2dOpUnnniCa6+9ltKlSwNQunRphg4dyuOPP860adNCVqSIBEfpIoZ/tbaZfKHNnPQALWa4bDqkkCUiEmxZBqyiRYtSrVq1016rXr06ERERuVaUiOSugdUtVvXL2CHQfIbL5O06kUFEJJiyDFjXXHMNL7/88im9dAKBAC+99BLXXHNNrhcnIrmndmnDyn4OfaoaLlvkccsyj+OeZrNERIIhy03uMTExpKSkUKtWLfr06UO5cuXYt28fs2fP5tixY7Rt25axY8cCGZ9auvXWW//yxbZt28bgwYPZv38/pUuX5q233qJu3bqn3Ddv3jxGjx6N7/ucOHGCO++8U60gRHJByQjDpE42bcr53LbcZ9WPASZfaFO5hDa/i4icCxPIot2zZZ31MYUYY/A87y/v69y5M1dffTVDhgxhypQpPPPMMyxbtizTPYFAgNjYWBYvXkzDhg1JTU2ldu3a/Pjjj5QqVSrTvSkpKTRt2pQ1a9bQpEmTs673vx08eJCYmJgcPVbynsYveJbt9Rm40OOED//ubNOp0tn/DsgpjV/40xiGN41f7slyBsv3g7snY9++faSkpDB//nwAkpOTGTlyJKmpqSQkJJxy/08//QTAL7/8wnnnnUeRIkWyfO6RI0cSHR1N//79SU5OzlZdhw4dytb9kr9o/IInKQIWdjQMW1mULnMC3Fv/OKMSj5ObnRw0fuFPYxjeNH7B8+eg+tet2oMkPT2dSpUqnWxgaowhPj6etLS0TAHLGMPkyZPp378/JUqU4NChQ0ybNu20x/T8Ydy4cTmewYJT/1AkvGj8gicGWHxxgPvX+Dy8tgjrfi3K2x1toiNzL2Vp/MKfxjC8afxyxxnXAA4cOMDDDz9M165dadq0KV27duWRRx7hwIEDOXqxPzc1PN3qpOu6/POf/2TmzJns3LmThQsXMnjwYA4ePJij1xSR7LEtw2PNbWZ2s/lsT4Bm013WH9DmdxGR7MgyYG3fvp2GDRvy2GOP4bouiYmJuK7Lo48+SsOGDdm+fXu2XiguLo5du3bhui6QEa7S09OJj4/PdN/atWvZvXs3F1xwAQDNmzenUqVKrFu3Lrs/m4icg4urWqy5xKGEA61muryzVa0cRETOVpYB6/bbbyc6Oppt27axePFi3n//fRYvXsx3331H6dKlueOOO7L1QuXKlaNx48ZMnDgRyGhkmpCQcMr+qz+C2JYtWwD47rvv2L59O4mJidn80UTkXNWIMizr63BZDcPgJR7XL/U4plYOIiJ/Kcs9WIsXL2bChAmnzDDFxcXx4IMPct1112X7xcaPH8+QIUMYM2YMUVFRvP322yev9erVi4cffphmzZoxfvx4BgwYgGVZJ/tuVa5cOduvJyLnrphjeKO9TZvyAUZ+6bFmf4ApXWyqllIrBxGRrJzxU4R/bEg/5UGOk6NPGSYlJZ3SluEPc+fOPfnvV1xxBVdccUW2n19EcocxhmG1DY3PgwELPJpMd5nUyaZ7XO63chARCUdZ/na84IILeOSRR07ZXH7o0CEee+wx2rZtm+vFiUj+0qysRcolDi3LGXp+4vHQGg//9K30REQKtSxnsJ5++mnatWtH1apV6dy5MxUqVGDv3r0sXLiQyMjITMt7IlJ4xBQ1fNTd5tGvfR5c47N8X4CJnWzOK6olQxGRP2Q5g1W/fn2++eYbhg0bxp49e1i0aBF79uxh+PDhrFu3jnr16oWyThHJRyxjuL+Jzcc9bFb+GKDpdJfVP+pThiIifzjtDNbRo0e58847+dvf/nbyvEERkT/rHmeRcolh4EKPC2Z5vNAGhtU2p/S8ExEpbE47g1W0aFHefPNNjhw5Eup6RCTMVC1lWNrHZmiSxYgvPK5Z4nHY1b4sESncslwibNOmDStWrAhlLSISporYhpfa2rzT0Wby9xnd37/4j5YMRaTwynKT+8MPP8ygQYNwHIeePXtSrly5U6b9dX6RiPy3v9WyaHyeYejnHu1mewyvHeDxFhZlimjJUEQKFxM43YGAgGX9/+RWVvspPM/LnarOUkpKCk2bNmXNmjU5Puz54MGDCophTOOXP3l+gFc2+fxjlU9xB/7V2uay6qfuzdL4hT+NYXjT+OWeLGew3njjDW1UFZEcsS3DTfVs+iVY/H2ZxxWLPN7eanjpAptqUfq9IiIFX5YBa8iQISEsQ0QKosolDFO6OMze6XPTlx71prg80MTitoYWEZaClogUXFlucq9evTrr1q077bUNGzZQvXr1XCtKRAqWPlUtNg50uL6OxejVPk2nuyzfq03wIlJwZRmwUlNTOXbs2GmvHT58mPT09FwrSkQKnpIRhrGtbVb1c4i0DG1medz5dRF+Pq6WDiJS8GRaIjx69CiHDx/mj33vv/zyyylnER49epQZM2ZQqVKl0FUpIgVGk1jDir424zb6/M+qCD7+0OX51jbJ1dSgVEQKjkwB64knnuDhhx8GMj452L179ywf+OCDD+ZqYSJScNmW4e/1bTqX/oV7N5Zi4EKP3nGGFy+wqVpKIUtEwl+mgNWvXz8SEhIIBAJce+213HvvvdSoUSPTAyIjI6lTpw7nn39+SAsVkYKncvEAM7s5zEj1GfmVR90pLo80tRhV38LRJngRCWOZAlajRo1o1KgRkDGD1bt3b2JjY/OkMBEpPPolWHSuZLhvtc8dK3wmfufzajubZmWz3CYqIpKvZfnba/DgwQpXIhIyUZGG59rYrOhnEwBazvT4+1cev2oTvIiEoSwD1pEjRxg9ejSJiYkUL14c27Yz/eM4WbbQEhHJseZlLVb1c3iyhcXrW3zqTHFZppYOIhJmskxJN910E5MmTWLgwIFcc801REZGhrIuESnEHMtwe0ObAdUsrlrs0XWux9we0L6ilgxFJDxkGbBmz57NU089xc033xzKekRETqpayjCvp03f+R49PvaY1R26VFbIEpH8L8vfVLZtk5SUFMpaREROUSLCMLu7TcdKhovmecxN03KhiOR/WQas66+/nnfffTeUtYiInFYxxzC9q02PKoZ+n3rMTFXIEpH8LcslwhIlSrB06VJat25N165dKV26dKbrxhhuvfXWXC9QRASgiG34sIvNVYs8BizweK8TXFpDy4Uikj9lGbDuvvtuANLS0lixYsUp1xWwRCTUIizDpM42Q5Z4XLHY47gPg2opZIlI/pNlwPJ9TcGLSP7jWIa3O9gUsTyu/iwjZF2bpJAlIvmLmlmJSNixLcNr7W2K2D5DP/c45gW4oa6d12WJiJyU6a99Y8aMYc+ePZlu+Pzzz/n9998zfe/777/n2muvzf3qRESyYBnDixdY3FLf4sYvfZ79xsvrkkRETsoUsO677z7S09NPfu15Hp06dWLLli2ZHvTjjz/y9ttvh6ZCEZEsGGMY28rinkYWty33eXytQpaI5A+ZlggDgVPP/Drd90RE8gtjDGOaWxSx4R+rfI568EATC2NMXpcmIoWY9mCJSNgzxvBgU5siNoxe5XPMgzHNFbJEJO8oYIlIgfGP822K2nDb8oyZrLGtFLJEJG+cErBO98tIv6BEJFzc2iBjJuumLzNmssZdYGHpd5iIhNgpAatTp05YVuaeMu3atcv0PfXIEpH87Ma6NkUsw7ClHsf9AOPb2tiWQpaIhE6mgPXAAw/kVR0iIkE1tLZFpA1Dlnik7HfpU9WiW2VDi3KGCIUtEcllClgiUmD9rZZF+WLw+mafF771eTgFoiKgcyVDtyqGblUsakQpbIlI8GmTu4gUaN2qWHSrYuH5AdbsDzB/V4D5PwQY9ZWPG/CpVgq6VbboVsXQuZKhdBEFLhE5dwpYIlIo2FbG8mCLcnBvE/jleIDP9gT4dFeA+T/4jN8MloGWZTNmt7pWNrQqZ7R3S0RyRAFLRAqlqEjDxVUNF1cFsEn9NcCnPwSYv8vnuQ0+D6VA87KGqV1s4koqZIlI9ugIehERIKGUYVhtiw+7OOz/m8PCXjZ7jwRoOt1l8W59clpEskcBS0TkT2zL0Lmyxep+Dg1jDF3neoxd7+noMBE5a9kKWKmpqSxYsICDBw/mVj0iIvlG2WKGT3ra3N7A4vYVPlcs8vj9hEKWiPy1LAPW7bffzi233HLy6+nTp5OUlES3bt2oVasWa9asCUmBIiJ5ybEMTxMuiWMAACAASURBVLS0mXyhzUdpAVrNdPnuZ4UsETmzLAPW9OnTadas2cmvR48eTa9evVi/fj0tWrTg3nvvDUmBIiL5wcDqFiv6OhzzoNkMlzlp2pclIlnLMmDt2bOH+Ph4ALZv386WLVu49957qV+/PqNGjWL16tUhK1JEJD+oF2NYdYlDh4qGi+Z5PLTGw9e+LBE5jSwDVnR0NPv27QPg008/JSYmhqZNmwIQGRnJkSNHQlOhiEg+Eh1pmN7V5pGmFg+l+PSd7/HTMYUsEcksyz5Y7du35/7772fv3r08/fTT9OvX7+S1LVu2nJzdEhEpbCxjuLeJTZNYw1WLPZrPcJne1aF+jPpliUiGLGewnn32WSpUqMA999xDfHw8jz322Mlr7777Lu3atQtJgSIi+VWveIvVlzgUc6DlTJcPtmtflohkyHIGq3LlyixatOi01+bNm0fRokVzrSgRkXBRI8qw7GKHYUs9Ll/kserHAI+3sHB0xI5IoZajPliu6xIZGZlbNYmIhJUSEYb3OtmMbWXxrw0+3eZ6/HhE+7JECjP1wRIRCQJjDLc2sFnQy2bDoYwjdlb9qCVDkcJKfbBERIKoYyWLlEscKhY3tJ3lMWGzQpZIYaQ+WCIiQValpOHzPjaDEw3XLfW4fqnHMU9LhiKFifpgiYjkgiK24dV2Dq+2s3lzq0/Hjzx++F0hS6SwUB8sEZFcNKy2RaMYSF7g0WS6y4cX2rSvmK3PF4lIGFIfLBGRXNainMWaSxzqlDZcOMfj+Q0eAR2xI1KgqQ+WiEgIlCtmWNDL5u6VPn9f5rNiX4BX29mUiFC/LJGCKMuA9YfDhw/z9ddfc/DgQWJiYmjSpAlRUVGhqE1EpEBxLMMzrWyalzUM/dzjm0Mu07o41IxWyBIpaM64EeCxxx6jQoUKtG/fnr59+9KuXTvKly/PmDFjQlWfiEiBc3kNixV9HY660GyGy0c71cpBpKDJMmA999xz3HfffVx55ZUsWrSITZs2sXjxYq666iruv/9+nn/++VDWKSJSoNSPMay6xKFDRUOf+R73r/bwfO3LEikoslwifPHFF7nzzjt54oknTn4vKSmJDh06EBUVxbhx4xg1alRIihQRKYiiIw3Tu9r8c63Pfat9Vv0Y4L1ONjFFtWQoEu6ynMFKS0uja9eup73WpUsX0tLScq0oEZHCwjKG/2ls83EPm5U/Bmg2w+Xr/ZrJEgl3WQasSpUq8cUXX5z22pdffkmlSpWy/WLbtm2jTZs2JCYm0qJFCzZu3Hja+44dO8bIkSOpVasW9erVY9CgQdl+LRGRcNI9LqOVQ5ki0GaWyztbtS9LJJxluUR43XXX8cADD3Ds2DEuvfRSKlSowN69e5k8eTJPP/00Dz30ULZfbMSIEQwfPpwhQ4YwZcoUhg4dyrJly06575577sGyLLZu3Yoxhj179mT7tUREwk1CKcMXfRxu+tJj8BKPFT8GeLaVRaStJUORcGMCWXS7CwQC3HHHHYwbNw7XdU9+33EcRo0axVNPPZWtF9q3bx+JiYns378fx3EIBAJUrFiR5cuXk5CQcPK+33//ncqVK7Nr1y5Klix5xudMSUmhadOmrFmzhiZNmmSrnj/80X5CwpPGL7xp/E4vEAjw2uYAN3/l0TTW8GEXm8ol8mfI0hiGN41f7jntDFYgEODgwYM88sgjjB49mhUrVnDo0CFiYmJo0aIF5513XrZfKD09nUqVKuE4GS9pjCE+Pp60tLRMAWv79u2cd955PProoyxYsIBixYrx4IMPcuGFF2b53CNHjiQ6Opr+/fuTnJycrboOHTqU7Z9F8g+NX3jT+GVtQHmo1sFiyPJiNJ7qMaHlUS4o6+V1WafQGIY3jV/w/DmonjZgua5LhQoVmDFjBr1796ZXr15BeXFjMv8N7HSTZydOnOD777+nbt26PP7446xbt44uXbqwceNGypYte9rnHTduXI5nsODUPxQJLxq/8Kbxy1rXGPi6coDLF3pcsrQ4T7awuLWBdcrv0rymMQxvGr/ccdpN7hEREVSuXBnPC97fluLi4ti1a9fJ5cZAIEB6evoph0ZXrVoVy7K46qqrAGjUqBHVqlXj22+/DVotIiLholwxw/xeNrc3sLh9hc+ABR47f9WnDEXyuyw/RThy5EieffZZjh07FpQXKleuHI0bN2bixIkATJ06lYSEhEzLgwCxsbFceOGFzJs3D4CdO3eyY8cOkpKSglKHiEi4cSzDEy1tpnSx+WJvgMTJLrct89h/VEFLJL/K8lOEO3fuZMuWLcTHx9OxY0fKlSuXaVraGMNzzz2XrRcbP348Q4YMYcyYMURFRfH222+fvNarVy8efvhhmjVrxiuvvMK1117L3XffjW3bvPrqq1SsWDEHP56ISMGRXM2iexXDs9/4PLXe5/UtPnc2zFg2LKlDo0XylSw/RVitWrUzP9AYvv/++1wp6mzpU4Si8QtvGr+c2380wJivfV7c6FO6CNzX2GJ47dC3dNAYhjeNX+7JcgZrx44doaxDRESyIbaoYWxrm1saWDywxuPvy3zGfuPzSDObK2oYrHy2EV6ksMlyD5aIiOR/8SUNb3ZwWN/foWGMYdBij8bTXOam+af9pLaIhEamgPX9999To0YN5syZk+UD5syZQ40aNdi8eXOuFyciImenXoxhRjeHLy+2iY409J7n0fEjj2V7deSOSF7IFLCeeeYZqlevTu/evbN8QO/evalVqxZjx47N9eJERCR72pS3WHKRzZzuNj8dD9Bmlke/+S4bD2k2SySUMgWsOXPmMHjw4L980N/+9jc+/fTTXCtKRERyzhhDr3iLr/s7TOxks/5ggAZTXa5Z4pL2m4KWSChkClh79uw5pS/V6VStWpXdu3fnVk0iIhIEljFcVdNi80CH51pbzE0PUOsD9dASCYVMAatUqVLs27fvLx+0b98+SpUqlWtFiYhI8ETahpH1bLZf5nBvY4vXt/jU+LfLoykev51Q0BLJDZkCVvPmzXn//ff/8kHvv/8+zZs3z7WiREQk+EpGGO5rkhG0rk2yeORrn5ofuEzY7OP5CloiwZQpYN18881MnTqV++6777TnEHqex/3338+0adMYNWpUyIoUEZHgKVvM8Gxrm62XOlxYyXDdUo9mM1w+261PHIoES6ZGo7169WL06NE89thjTJgwgS5duhAXF4cxhrS0NBYsWMDevXsZPXo0PXv2zKuaRUQkCKqWMrzX2WFkPZ9bl/t0muPRP8HnyZY2NaLUqFTkXJzSyf3RRx+lbdu2PP3000yZMoWjR48CULRoUdq2bcubb75J9+7dQ16oiIjkjtblLb662PDv7QHuXulR90OXv9e3+J/GFtGRCloiOXHao3J69OhBjx498DyPAwcOEAgEiI2NxbbtUNcnIiIhYBnDlTUN/RIMT6/3eWKdz1tbfR5pZnFdkoVtKWiJZMcZj8qxbZty5cpRvnx5hSsRkUKguGO4v0nG/qyecYbrv/BpPN1l4Q/anyWSHTqLUERETlG5hOHtjg4r+9mUijB0mevRd77Ltp/1aUORs6GAJSIiWWpe1uKLPjb/7myz7kCAelMyGpUeOqagJXImClgiInJGxhguq2GxaaDDg00sXt3sU+sDl5c2erhaORQ5LQUsERE5K8Ucw+jGNtsuc7i4qmHklz7tFxTn/e98jrqa0RL5bwpYIiKSLRWLG97o4LD6EofyRQNcudij8iSXW5d5bDiooCUCClgiIpJDTWIN09sfYdPAjKN33vvOp8FUl9YzM47f0TmHUpgpYImIyDmpXdrwVEubXVc6TOliEx0Jw5Z6VHzPZfhSl5X7fAIBhS0pXE7baFRERCS7Im1DcjVDcjWLnb8GeHOrzxtbfF7b7NEgBoYlWVxV0yKmqJqWSsGnGSwREQm6qqUMDza12XG5w9weNrWiDLct96k0yeWqRS6Ld/v4mtWSAkwzWCIikmtsy9AzztAzzmLv4QDvbPN5fYvPpDkeNaLguiSLwYkWFYtrVksKFs1giYhISJQvbrizkc3mgQ5LLrJpU87wUIpP3CSXfvNd5qT5uL5mtaRgUMASEZGQMsbQvqLFO50cdl/l8Fxri9RfA1w0z6PhVJdv1OpBCgAFLBERyTNlihhuqmfzdX+HFX1tHAtazHB5fbM+eSjhTQFLRETynDGGFuUsVvR1uLqWxbClHoMWe/x6XCFLwpMCloiI5BvFHMP4djbvd7aZlRag2QyXtQcUsiT8KGCJiEi+c3kNi5RLHEo40Gqmy8sbPS0ZSlhRwBIRkXypVrThq4sdrkuyuPFLn8sXefysJUMJEwpYIiKSbxV1DOMusPnwQptP0gM0meay5keFLMn/FLBERCTfG1Dd4uv+DjFFDG1mubywQUuGkr8pYImISFioHmX44mKbG+pajFrmk7zA49AxhSzJnxSwREQkbBSxDf9qbTO9q83i3RlLhiv3+XldlsgpFLBERCTs9EuwWNvfoXxxwwWzPMau15Kh5C8KWCIiEpaqljJ8fpHN3+tb3L7Cp+98j4NHFbIkf1DAEhGRsBVpG55uZTO7m82XewM0nOYya6eWDCXvKWCJiEjYu6hqxpJhoxhD3/keAxe47Dms2SzJOwpYIiJSIMSVNHzU3WZSJ5slewLU+VCHRkveUcASEZECwxjDFTUtNg10uCTBMGypR6c5Hlt/UsiS0FLAEhGRAue8ooY3Ozgs6GWz6/eMvVmPfe1x3FPQktBQwBIRkQLrwsoW65Mdbqlv8cAan2YzXFaob5aEgAKWiIgUaMUdw+MtbFb1c4i0DK1nevz9K49fdXC05CIFLBERKRQaxxqW97V5uqXF61t86k1xmZOm2SzJHQpYIiJSaDiW4baGNhuSHeqUNlw0z+OKRS571dJBgkwBS0RECp1qUYZPetq829Hm010B6kxxeXOLWjpI8ChgiYhIoWSMYVAti82XOlwUZ7j2c4+ucz22/6KQJedOAUtERAq12KKGdzo5zOtps/2XAPWnuDyx1uOEr6AlOaeAJSIiAnSrYrFhgMNNdS1Gr/ZpMcNl/QGFLMkZBSwREZH/UyIi4/DolX0dvAA0m+Hyz7UermazJJsUsERERP6kaVnDqn4OtzewuHe1T7vZOm5HskcBS0RE5DSK2IZ/trBZ2sdm/9EA509zeWGDh69PGspZUMASERE5gzblLdb2dxiaZDFqmU/XuR47f1XIkjNTwBIREfkLJSIML1xgs6CXzbafAzSYqr5ZcmYKWCIiImfpwsoW3wxwSK6W0Ter73yP/6gLvJyGApaIiEg2REca3uzgMLObzYofM/pmffi9zjSUzBSwREREcuDiqhbfDnDoVMlw6UKPKxe5HDyq2SzJoIAlIiKSQ7FFDZMvtHmvk83H6QHqT3X5OF2zWaKAJSIick6MMVxZM6MLfKMYQ69PPIYvdfn1uGazCjMFLBERkSCoXMIwt4fN+LY2k74L0Giay+d7NJtVWClgiYiIBIkxhuF1LNYnO1QpYej4kcfVi13W/KjZrMJGAUtERCTIqkcZFve2eb6NxZL/BGg2w+WCWS7/3u5zQucaFgohC1jbtm2jTZs2JCYm0qJFCzZu3HjG+x966CGMMWzYsCFEFYqIiASPbRlG1rPZfpnDtC42RSy4YpFHwvsuj6Z47DuioFWQhSxgjRgxguHDh7N161buuusuhg4dmuW9KSkpLF++nPj4+FCVJyIikiscy3BJNYtFFzmsT3a4KN5izFqfuEkugz9zWf2j9mkVRCYQgj7/+/btIzExkf379+M4DoFAgIoVK7J8+XISEhIy3Xvs2DE6duzIpEmT6NSpEx999BH169c/7fOmpKTQtGlTWrduTXR0NP379yc5OTlbtR06dIgyZcrk9EeTPKbxC28av/CnMcyZQ8fhvdQIJmyPJO2wRbMYj+E1j9OnsktkCDfvaPyCJyYmJtPXTiheND09nUqVKuE4GS9njCE+Pp60tLRTAtb999/PoEGDqFat2lk//7hx42jSpEmO6/vzH4qEF41feNP4hT+NYfbFAPdXgP9pEeCjtAAvfGsYvtKmQjG4oa7F8NoWFYqb0NSi8csVIcvJxmT+D+V0E2fLli1j1apV3HjjjaEqS0REJM/YlqFvgsWC3g4bkh36JVg8sc4n/n2XQYtdVu7T8mG4CknAiouLY9euXbiuC2SEq/T09FP2WC1ZsoTNmzdTrVo1EhIS2LVrF927d+fjjz8ORZkiIiJ5pl6M4eW2NruudHi8ucVXewO0nOnRcobL9B0KWuEmJAGrXLlyNG7cmIkTJwIwdepUEhISTlkevOeee9i9ezepqamkpqZSpUoV5s2bR8+ePUNRpoiISJ4rU8RwW0ObbZc6zOpmUyoC+i/wuH25h6cWD2EjZEuE48ePZ/z48SQmJvL4448zYcKEk9d69erF6tWrQ1WKiIhIvmdbhj5VLT7tZfN8a4t/bfC5aJ7HzzqCJyyEZJM7QFJSEsuWLTvttblz5572+6mpqblYkYiISP5njOHm+jZJpQ2XLvRoNdNldjeHmtGh2QQvOaNO7iIiImGgWxWLFX0d/AC0mOmy6Afty8rPFLBERETCRFJpw/K+Ds1iDd0+9nh5o5fXJUkWFLBERETCSJkihrk9bG6qa3Hjlz43fenpfMN8KGR7sERERCQ4HMvwXBubemUMN33psfmnAB9eaBNTVPuy8gvNYImIiISp4XUyPmW49kCAljNdNh3STFZ+oYAlIiISxjpWsljVzyHSglYzXT5J1+b3/EABS0REJMxVjzIs6+vQroKh9zyPf33jnfZIOgkdBSwREZECICrSMLObze0NLG5d7jNsqcdxTyErr2iTu4iISAFhW4YnW2Zsfh++1GPrzx5Tu9iULabN76GmGSwREZECZnCixeKLbLb8HKD5DJdvDmomK9QUsERERAqgNuUzNr+XjoQ2s1xm7dTm91BSwBIRESmg4ksavrjYoWtlQ7/5HsOXunyc7nPE1YxWbtMeLBERkQKsZIRhShebJ9f5jN/k89pmj6I2dKpk6BATwcDaAapHaY9WsClgiYiIFHCWMdxzvs3djSw2/wQfp/vMTQ9w3/oi3LPOJSkaesVZ9IwztK9oKGIrcJ0rBSwREZFCwhhDnTJQp4zNbQ1h596DfH2kNHPTfSbv8Hl2A5Rw4MLKhl5xhp5xFvElFbZyQgFLRESkkCoVAf3KW/RLsAgEAmw4BHPTMma3bvrSxwv41C8DPeMsesUZLqhgiLAUuM6GApaIiIhgjKFBDDSIsbn7fPjpWIBPfwgwN93nnW0+T62HqAjoWsVwcbzFVTUNtsJWlhSwRERE5BSlixgGVjcMrG7hBwKsPfD/s1tDlni8tdUwsZNNpRIKWaejNg0iIiJyRpYxNIk13NvE5qu+Dot6ZzQxbTQto+2DnEoBS0RERLKlYyWLdckOLcsaen3iccdynXv4ZwpYIiIikm2xRQ2zu9s809Li+W992s72+P4Xhaw/KGCJiIhIjhhjuK2hzZcX2xw4GuD8aS4fbNeSIShgiYiIyDlqXtbi6/4OveMNly/yGPa5y+FCfhyPApaIiIics6hIw6RONq+3s3nvuwDNp7tsOFh4Q5YCloiIiASFMYahtS1WX+JgGWg+w+XVTT6BQOELWgpYIiIiElR1yxhW9nMYXMtixBcely30+OlY4QpZClgiIiISdMUcwyvtbCZfaDNvV4DG01xW7Cs8G+AVsERERCTXDKxusba/Q7lihrazPJ5a5+EXgiVDBSwRERHJVdWiDEv72NzawOKulT69P/HYd6RghyydRSgiIiK5LtI2PNnSpnMlw9WfeTSa6nJ/E4uyRQ0lI/i/fwwlHU5+XdzJOKYnHClgiYiISMj0iLNYl2wY8pnHjV/+9Z6sEv8VuDLClznl6yaxhstrGIo7+SeMKWCJiIhISFUsbpjXy+GEH+D3E/DbCfjNhd9OBDL+/eTX//+9X/90z6/HYc9h+OW4z0sb4c4VMKy2xY11LeJL5n3QUsASERGRPBFhGUoXgdJF/vhOzoLR9l8CvPitz8sbfZ5a73NJgmFUPYt2FQwmj5YYtcldREREwlqNKMPY1jY/XOXwQhuLbw8F6PCRx/nTXCZs9jmSB8f2KGCJiIhIgVAywnBjXZuNAxzm97SJL2kYttSjyiSXf6z0SPstdEFLAUtEREQKFGMMXatYzO7usO2yjI7yL230qf5vlwELXD7fk/vH9yhgiYiISIH138uHz//X8mHjaS5vbMm95UMFLBERESnw/lg+/HaAw7yeNnElDdd97hH3f8uH6UFePtSnCEVERKTQsIyhWxVDtyoW3/0c4MWNPi9t9Nn1e4B3OwUvFilgiYiISKFUM9rwbGubh5ta/OYG97kVsERERKRQKxVpKBUZ3OfUHiwRERGRIFPAEhEREQkyBSwRERGRIFPAEhEREQmyQh+wpk6dmtclyDnQ+IU3jV/40xiGN41f7in0AWvatGl5XYKcA41feNP4hT+NYXjT+OWesG7TcOTIEQA2bdqU4+f4+eefSUlJCVZJEmIav/Cm8Qt/GsPwpvELrtq1a1O8eHEATCC3TzvMRe+99x6DBg3K6zJEREREWLNmDU2aNAHCPGDt37+fefPmkZCQQLFixfK6HBERESnECswMloiIiEh+VOg3uYuIiIgEmwKWiIiISJApYImIiIgEmQKWiIiISJCFdR+sYDl27BgjRowgKioKz/N48cUXz+qa5A9nGqPU1FR69uxJhw4dqFChAg8++GDeFSqndejQIe644w7mz59Penp6pmt6/4WHM42h3oP535IlS3jnnXc4evQoZcqUYdy4cSev6T2YcwVuBmvUqFEkJCRgjGHDhg2Zrm3bto02bdqQmJhIixYt2LhxI5DRybZDhw48//zzlClThmXLlp18zJmuSfAFe/wAoqOjOXr0KNWqVQvZz1FY5WT8ypQpw4QJE0hKSjrl+fT+C71gjyHoPRhKORm/Dh06MGHCBN577z3S0tL49ddfTz5G78GcK3ABa8CAAXzxxRdUrVr1lGsjRoxg+PDhbN26lbvuuouhQ4cCsHPnThISEgCoXr06O3fuPPmYM12T4Av2+FWtWpXly5fz5ptvMmvWLFJTU0PxYxRaORm/M9H7L/SCPYZ6D4bWuYzf3LlzqVOnDqVKlTr5Pb0Hc67ABaz27dtTpUqVU76/b98+UlJSTnZ+T05OZseOHaSmphIXF3fyP5rU1FTi4+NPPu5M1yT4gj1+xpiT/1u+fPlMfzOT4MvJ+J2J3n+hF+wx1HswtHI6fq+99horV67kiSeeyPQ4vQdzrtDswUpPT6dSpUo4TsaPbIwhPj6etLQ0kpOTuf7661m/fj1HjhyhTZs2bNmyhfHjxzNmzJhTrkno5XT8+vTpwzvvvANA6dKladCgQV7+GIXWmcYvISGB66+/ns2bN3P99ddz55134rqu3n/5TE7HUO/B/OFM47dmzRoefvhhevfuzfXXX88jjzzCwYMH9R48R4UmYMH//03qD380sS9atChvvfVWpmtJSUmMHTsW4JRrkjdyOn6dOnUKSX1yZlmNH8Arr7xyyv16/+U/OR1DvQfzh6zGLzk5meTk5EzXypYtq/fgOSpwS4RZiYuLY9euXbiuC2T8h5Wenq7pzjCh8QtvGr/wpzEMbxq/0Cs0AatcuXI0btyYiRMnAjB16lQSEhJObt6T/E3jF940fuFPYxjeNH55IFDA3HjjjYHKlSsHbNsOlC9fPlCjRo2T1zZv3hxo1apVoFatWoGmTZsGNmzYkIeVyulo/MKbxi/8aQzDm8Yv/zCBwH8toouIiIjIOSs0S4QiIiIioaKAJSIiIhJkClgiIiIiQaaAJSIiIhJkClgiIiIiQaaAJSIiIhJkClgiIiIiQaaAJSIiIhJkClgiItnw4IMPUrJkybwuQ0TyOQUsERERkSBTwBIREREJMgUsEcn3li1bRufOnSlRogTR0dFceeWV7Nu3D4DU1FSMMbz99tsMHTqU6OhoYmJiuO2223BdN9PzbNiwgR49elCyZEmioqLo27cv3333XaZ7fN9n7Nix1KlThyJFilChQgUGDhzIzz//nOm+9evX07ZtW4oXL079+vWZN29epuuzZs2iWbNmlCxZktKlS9OsWTPmzp2bC386IpIfKWCJSL62bNkyOnbsSHR0NB988AGvvvoqq1at4uKLL8503+jRo/F9n8mTJ3PnnXfywgsvcO+99568np6eTrt27di7dy9vv/02r7/+Olu3bqVdu3b8+OOPJ++7+eabueuuu7jooouYPXs2L774IqVKleK33347ec+JEycYNGgQQ4YMYfr06cTGxpKcnMyBAwcA2L59OwMGDKBevXpMnz6d/23n/kHa3MI4jn9fNYIplRprSbBSBdGpTl0a/IOxiC6lSrVRcAguYhdTOqkQoViQUrq4iIODLiLooAGhKFL/LIKLgkgpior/wIgOoUjS06H0hVR623ub0vTy+8ALOSfnPHneMz2cc8j4+DjNzc2cnZ395tUSkZRhRERSWGVlpfF6vebTp09238bGhrEsy4TDYbO9vW0AU1FRkTCvt7fXOJ1OE4lEjDHGBINB43Q6zcnJiT1mZ2fHOBwOEwqFjDHGbG1tGcuyzMuXL7+bTygUMoAJh8N23/v37w1gRkdHjTHGTExMGMBcXFz88vuLyN9JO1gikrKi0SjLy8s0NTURj8eJxWLEYjFKS0vxeDysrq7aYxsaGhLmNjY2Eo1GWV9fB2BxcRGfz0deXp495s6dO3i9XhYXFwGYn5/HGEN7e/s/5pWWlsaDBw/sdnFxMZmZmezv7wNQVlZGeno6ra2tTE9PXzleFJH/PxVYIpKyzs7OiMfjBINBHA5HwnNwcMDe3p499tatWwlzv7YPDw/tWG63+8pvuN1uIpEIAKenp2RkZFyJ9a2srCwyMzMT1pP4JgAAAj5JREFU+hwOBx8/fgSgpKSEmZkZzs/PaWhoIC8vj4cPH7K7u/svV0BE/lYZfzoBEZHvuXHjBpZl0d3dzaNHj658f/PmTfvz10vv37Y9Hg8ALpeL4+PjKzGOjo5wuVwA5ObmEovFODk5+WGR9SN1dXXU1dVxcXHB7OwswWCQQCDA3NzcL8UVkb+DdrBEJGVdu3aN+/fvs7m5yb179648hYWF9tipqamEuZOTkzidTu7evQtAeXk5c3Nz9kV0+HLxfWVlhYqKCgB8Ph+WZTEyMpK0d8jOzqa5uRm/38/m5mbS4opIatMOloiktFevXuHz+Xjy5Al+v5+cnBz29/d5+/YtgUDALrI+fPhAIBDA7/eztrbGwMAAXV1d5OTkABAMBhkZGaG2tpaenh7i8TihUAiXy8XTp0+BL0d7HR0d9Pb2EolEqKmpIRqNEg6H6evrIz8//6dyHhoaYmVlhfr6ejweD9vb24yNjVFbW/tb1khEUo8KLBFJaV6vl6WlJUKhEIFAgMvLS27fvk1NTQ3FxcX2f1319/ezsLBAU1MT6enpdHZ20t/fb8cpKCjg3bt3PH/+nLa2NtLS0qiurub169cJF98HBwcpKipieHiYN2/ekJubS1VVFdevX//pnMvKypienubZs2ecnp7idrtpaWnhxYsXyVsYEUlpljHG/OkkRET+q52dHYqKipiYmODx48d/Oh0REUB3sERERESSTgWWiIiISJLpiFBEREQkybSDJSIiIpJkKrBEREREkuwze/XvFuavAeMAAAAASUVORK5CYII=", "text/html": [ - "" + "" ], "image/svg+xml": [ "\n", @@ -270,7 +284,7 @@ " \n", " \n", " \n", - " \n", " \n", @@ -278,10 +292,10 @@ " \n", " \n", + "\" id=\"mbfc883ff76\" style=\"stroke:#000000;stroke-width:0.5;\"/>\n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -339,13 +353,13 @@ " \n", " \n", " \n", - " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -387,13 +401,13 @@ " \n", " \n", " \n", - " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -409,13 +423,13 @@ " \n", " \n", " \n", - " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -431,13 +445,13 @@ " \n", " \n", " \n", - " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -636,18 +650,18 @@ " \n", " \n", " \n", - " \n", " \n", " \n", " \n", " \n", + "\" id=\"md549519618\" style=\"stroke:#000000;stroke-width:0.5;\"/>\n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -671,7 +685,7 @@ "z\n", "\" id=\"DejaVuSans-52\"/>\n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -680,13 +694,13 @@ " \n", " \n", " \n", - " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -723,7 +737,7 @@ "z\n", "\" id=\"DejaVuSans-54\"/>\n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -732,13 +746,13 @@ " \n", " \n", " \n", - " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -784,7 +798,7 @@ "z\n", "\" id=\"DejaVuSans-56\"/>\n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -793,18 +807,18 @@ " \n", " \n", " \n", - " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -940,30 +954,30 @@ " \n", " \n", " \n", - " \n", " \n", @@ -1003,7 +1017,7 @@ " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -1011,7 +1025,7 @@ ] }, "metadata": {}, - "execution_count": 7 + "execution_count": 8 } ], "cell_type": "code", @@ -1029,7 +1043,7 @@ " ylab = \"Cross Entropy\")" ], "metadata": {}, - "execution_count": 7 + "execution_count": 8 }, { "outputs": [], @@ -1038,7 +1052,7 @@ "savefig(\"iris_history.png\")" ], "metadata": {}, - "execution_count": 8 + "execution_count": 9 }, { "cell_type": "markdown", @@ -1056,11 +1070,11 @@ "file_extension": ".jl", "mimetype": "application/julia", "name": "julia", - "version": "1.5.1" + "version": "1.6.0" }, "kernelspec": { - "name": "julia-1.5", - "display_name": "Julia 1.5.1", + "name": "julia-1.6", + "display_name": "Julia 1.6.0", "language": "julia" } }, diff --git a/examples/iris/iris.jl b/examples/iris/iris.jl index db3680a7..d0f2ea2a 100644 --- a/examples/iris/iris.jl +++ b/examples/iris/iris.jl @@ -4,6 +4,8 @@ using Pkg Pkg.activate(@__DIR__) Pkg.instantiate() +# **Julia version** is assumed to be 1.6.* + using MLJ using Flux import RDatasets @@ -75,5 +77,5 @@ plot(curve.parameter_values, savefig("iris_history.png") using Literate #src -Literate.markdown(@__FILE__, @__DIR__, execute=true) #src +Literate.markdown(@__FILE__, @__DIR__, execute=false) #src Literate.notebook(@__FILE__, @__DIR__, execute=true) #src diff --git a/examples/iris/iris.md b/examples/iris/iris.md index d55a7cc7..70ad1e64 100644 --- a/examples/iris/iris.md +++ b/examples/iris/iris.md @@ -1,14 +1,18 @@ ```@meta -EditURL = "/iris.jl" +EditURL = "/../../MLJFlux/examples/iris/iris.jl" ``` # Using MLJ with Flux to train the iris dataset -```julia +```@example iris using Pkg Pkg.activate(@__DIR__) Pkg.instantiate() +``` + +**Julia version** is assumed to be 1.6.* +```@example iris using MLJ using Flux import RDatasets @@ -23,11 +27,6 @@ pyplot(size=(600, 300*(sqrt(5)-1))); nothing #hide ``` -``` - Activating environment at `~/Dropbox/Julia7/MLJ/MLJFlux/examples/iris/Project.toml` - -``` - Following is a very basic introductory example, using a default builder and no standardization of input features. @@ -36,33 +35,16 @@ example](https://github.com/FluxML/MLJFlux.jl/blob/dev/examples/mnist). ## Loading some data and instantiating a model -```julia +```@example iris iris = RDatasets.dataset("datasets", "iris"); y, X = unpack(iris, ==(:Species), colname -> true, rng=123); NeuralNetworkClassifier = @load NeuralNetworkClassifier clf = NeuralNetworkClassifier() ``` -``` -NeuralNetworkClassifier( - builder = Short( - n_hidden = 0, - dropout = 0.5, - σ = NNlib.σ), - finaliser = NNlib.softmax, - optimiser = ADAM(0.001, (0.9, 0.999), IdDict{Any,Any}()), - loss = Flux.Losses.crossentropy, - epochs = 10, - batch_size = 1, - lambda = 0.0, - alpha = 0.0, - optimiser_changes_trigger_retraining = false, - acceleration = CPU1{Nothing}(nothing)) @252 -``` - ## Incremental training -```julia +```@example iris import Random.seed!; seed!(123) mach = machine(clf, X, y) fit!(mach) @@ -70,13 +52,9 @@ fit!(mach) training_loss = cross_entropy(predict(mach, X), y) |> mean ``` -``` -0.8993467f0 -``` - Increasing learning rate and adding iterations: -```julia +```@example iris clf.optimiser.eta = clf.optimiser.eta * 2 clf.epochs = clf.epochs + 5 @@ -84,43 +62,19 @@ fit!(mach, verbosity=2); nothing #hide ``` -``` -┌ Info: Updating Machine{NeuralNetworkClassifier{Short,…},…} @545. -└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/4DmTL/src/machines.jl:342 -┌ Info: Loss is 0.853 -└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/wj7HX/src/core.jl:122 -┌ Info: Loss is 0.8207 -└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/wj7HX/src/core.jl:122 -┌ Info: Loss is 0.8072 -└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/wj7HX/src/core.jl:122 -┌ Info: Loss is 0.752 -└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/wj7HX/src/core.jl:122 -┌ Info: Loss is 0.7077 -└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/wj7HX/src/core.jl:122 - -``` - -```julia +```@example iris training_loss = cross_entropy(predict(mach, X), y) |> mean ``` -``` -0.7076617f0 -``` - ## Accessing the Flux chain (model) -```julia +```@example iris chain = fitted_params(mach).chain ``` -``` -Chain(Chain(Dense(4, 3, σ), Dropout(0.5), Dense(3, 3)), softmax) -``` - ## Evolution of out-of-sample performance -```julia +```@example iris r = range(clf, :epochs, lower=1, upper=200, scale=:log10) curve = learning_curve(clf, X, y, range=r, @@ -133,9 +87,8 @@ plot(curve.parameter_values, xscale=curve.parameter_scale, ylab = "Cross Entropy") ``` -![](3397330029.png) -```julia +```@example iris savefig("iris_history.png") ``` diff --git a/examples/iris/iris_history.png b/examples/iris/iris_history.png index 7c945865..8c7ca1e1 100644 Binary files a/examples/iris/iris_history.png and b/examples/iris/iris_history.png differ diff --git a/examples/mnist/Manifest.toml b/examples/mnist/Manifest.toml index 85a21a08..5886b42e 100644 --- a/examples/mnist/Manifest.toml +++ b/examples/mnist/Manifest.toml @@ -49,10 +49,10 @@ uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" version = "0.4.1" [[CUDA]] -deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CompilerSupportLibraries_jll", "DataStructures", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "MacroTools", "Memoize", "NNlib", "Printf", "Random", "Reexport", "Requires", "SparseArrays", "Statistics", "TimerOutputs"] -git-tree-sha1 = "6893a46f357eabd44ce0fc1f9a264120a1a3a732" +deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CompilerSupportLibraries_jll", "DataStructures", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "MacroTools", "Memoize", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "TimerOutputs"] +git-tree-sha1 = "364179416eabc34c9ca32126a6bdb431680c3bad" uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" -version = "2.6.3" +version = "3.2.1" [[Cairo_jll]] deps = ["Artifacts", "Bzip2_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] @@ -61,22 +61,22 @@ uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a" version = "1.16.0+6" [[CategoricalArrays]] -deps = ["DataAPI", "Future", "JSON", "Missings", "Printf", "Statistics", "StructTypes", "Unicode"] -git-tree-sha1 = "f713d583d10fc036252fd826feebc6c173c522a8" +deps = ["DataAPI", "Future", "JSON", "Missings", "Printf", "RecipesBase", "Statistics", "StructTypes", "Unicode"] +git-tree-sha1 = "1562002780515d2573a4fb0c3715e4e57481075e" uuid = "324d7699-5711-5eae-9e2f-1d82baa6b597" -version = "0.9.5" +version = "0.10.0" [[ChainRules]] deps = ["ChainRulesCore", "Compat", "LinearAlgebra", "Random", "Reexport", "Requires", "Statistics"] -git-tree-sha1 = "a426b3526dffff05ef3eaab35d6dc2869ec5846a" +git-tree-sha1 = "3f1d9907dc8559cc7d568c5dd6eb1b583ac00aec" uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" -version = "0.7.60" +version = "0.7.65" [[ChainRulesCore]] deps = ["Compat", "LinearAlgebra", "SparseArrays"] -git-tree-sha1 = "44e9f638aa9ed1ad58885defc568c133010140aa" +git-tree-sha1 = "9b0375dc013ab0fc472b37cb8b18eed66b83f76b" uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" -version = "0.9.37" +version = "0.9.43" [[CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] @@ -86,21 +86,21 @@ version = "0.7.0" [[ColorSchemes]] deps = ["ColorTypes", "Colors", "FixedPointNumbers", "Random", "StaticArrays"] -git-tree-sha1 = "9d7dfad1326b1ad29afa1366587806a14d727745" +git-tree-sha1 = "c8fd01e4b736013bc61b704871d20503b33ea402" uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4" -version = "3.12.0" +version = "3.12.1" [[ColorTypes]] deps = ["FixedPointNumbers", "Random"] -git-tree-sha1 = "32a2b8af383f11cbb65803883837a149d10dfe8a" +git-tree-sha1 = "024fe24d83e4a5bf5fc80501a314ce0d1aa35597" uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" -version = "0.10.12" +version = "0.11.0" [[Colors]] deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] -git-tree-sha1 = "82f4e6ff9f847eca3e5ebc666ea2cd7b48e8b47e" +git-tree-sha1 = "417b0ed7b8b838aa6ca0a87aadf1bb9eb111ce40" uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" -version = "0.12.7" +version = "0.12.8" [[CommonSubexpressions]] deps = ["MacroTools", "Test"] @@ -110,9 +110,9 @@ version = "0.3.0" [[Compat]] deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] -git-tree-sha1 = "ac4132ad78082518ec2037ae5770b6e796f7f956" +git-tree-sha1 = "0900bc19193b8e672d9cd477e6cd92d9e7c02f99" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "3.27.0" +version = "3.29.0" [[CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] @@ -177,10 +177,10 @@ uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" version = "1.0.2" [[Distances]] -deps = ["LinearAlgebra", "Statistics"] -git-tree-sha1 = "366715149014943abd71aa647a07a43314158b2d" +deps = ["LinearAlgebra", "Statistics", "StatsAPI"] +git-tree-sha1 = "abe4ad222b26af3337262b8afb28fab8d215e9f8" uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" -version = "0.10.2" +version = "0.10.3" [[Distributed]] deps = ["Random", "Serialization", "Sockets"] @@ -188,9 +188,9 @@ uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" [[Distributions]] deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns"] -git-tree-sha1 = "e64debe8cd174cc52d7dd617ebc5492c6f8b698c" +git-tree-sha1 = "a837fdf80f333415b69684ba8e8ae6ba76de6aaa" uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" -version = "0.24.15" +version = "0.24.18" [[DocStringExtensions]] deps = ["LibGit2", "Markdown", "Pkg", "Test"] @@ -262,10 +262,10 @@ uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" version = "0.8.4" [[Flux]] -deps = ["AbstractTrees", "Adapt", "CUDA", "CodecZlib", "Colors", "DelimitedFiles", "Functors", "Juno", "LinearAlgebra", "MacroTools", "NNlib", "Pkg", "Printf", "Random", "Reexport", "SHA", "Statistics", "StatsBase", "Test", "ZipFile", "Zygote"] -git-tree-sha1 = "c443bf5a8329573a68364106b2c29bb6938dc6f5" +deps = ["AbstractTrees", "Adapt", "CUDA", "CodecZlib", "Colors", "DelimitedFiles", "Functors", "Juno", "LinearAlgebra", "MacroTools", "NNlib", "NNlibCUDA", "Pkg", "Printf", "Random", "Reexport", "SHA", "Statistics", "StatsBase", "Test", "ZipFile", "Zygote"] +git-tree-sha1 = "5e94fff7b4385fdd059863300b6b25ea0f849dda" uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c" -version = "0.11.6" +version = "0.12.3" [[Fontconfig_jll]] deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"] @@ -299,9 +299,9 @@ version = "1.0.5+6" [[Functors]] deps = ["MacroTools"] -git-tree-sha1 = "f40adc6422f548176bb4351ebd29e4abf773040a" +git-tree-sha1 = "a7bb2af991c43dcf5c3455d276dd83976799634f" uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" -version = "0.1.0" +version = "0.2.1" [[Future]] deps = ["Random"] @@ -309,21 +309,21 @@ uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" [[GLFW_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Pkg", "Xorg_libXcursor_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll"] -git-tree-sha1 = "bd1dbf065d7a4a0bdf7e74dd26cf932dda22b929" +git-tree-sha1 = "a199aefead29c3c2638c3571a9993b564109d45a" uuid = "0656b61e-2033-5cc2-a64a-77c0f6c09b89" -version = "3.3.3+0" +version = "3.3.4+0" [[GPUArrays]] -deps = ["AbstractFFTs", "Adapt", "LinearAlgebra", "Printf", "Random", "Serialization"] -git-tree-sha1 = "9c95b2fd5c16bc7f97371e9f92f0fef77e0f5957" +deps = ["AbstractFFTs", "Adapt", "LinearAlgebra", "Printf", "Random", "Serialization", "Statistics"] +git-tree-sha1 = "df5b8569904c5c10e84c640984cfff054b18c086" uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "6.2.2" +version = "6.4.1" [[GPUCompiler]] deps = ["DataStructures", "ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Scratch", "Serialization", "TimerOutputs", "UUIDs"] -git-tree-sha1 = "ef2839b063e158672583b9c09d2cf4876a8d3d55" +git-tree-sha1 = "42d635f6d87af125b86288df3819f805fb4d851a" uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" -version = "0.10.0" +version = "0.11.5" [[GR]] deps = ["Base64", "DelimitedFiles", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Printf", "Random", "Serialization", "Sockets", "Test", "UUIDs"] @@ -356,15 +356,15 @@ uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" version = "2.59.0+4" [[Grisu]] -git-tree-sha1 = "03d381f65183cb2d0af8b3425fde97263ce9a995" +git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2" uuid = "42e2da0e-8278-4e71-bc24-59509adca0fe" -version = "1.0.0" +version = "1.0.2" [[HTTP]] deps = ["Base64", "Dates", "IniFile", "MbedTLS", "NetworkOptions", "Sockets", "URIs"] -git-tree-sha1 = "c9f380c76d8aaa1fa7ea9cf97bddbc0d5b15adc2" +git-tree-sha1 = "b855bf8247d6e946c75bb30f593bfe7fe591058d" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "0.9.5" +version = "0.9.8" [[IOCapture]] deps = ["Logging"] @@ -401,9 +401,9 @@ version = "1.3.0" [[IterationControl]] deps = ["EarlyStopping", "InteractiveUtils"] -git-tree-sha1 = "afbb8ba60564b15c677e605c3c943a7ba1e72d99" +git-tree-sha1 = "f61d5d4d0e433b3fab03ca5a1bfa2d7dcbb8094c" uuid = "b3c1a2ee-3fec-4384-bf48-272ea71de57c" -version = "0.3.3" +version = "0.4.0" [[IteratorInterfaceExtensions]] git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" @@ -448,9 +448,9 @@ version = "3.100.0+3" [[LLVM]] deps = ["CEnum", "Libdl", "Printf", "Unicode"] -git-tree-sha1 = "b616937c31337576360cb9fb872ec7633af7b194" +git-tree-sha1 = "a220efe4a6bc1c71809d002eb9ed9209ce5a86fb" uuid = "929cbde3-209d-540e-8aea-75f648917ca0" -version = "3.6.0" +version = "3.7.0" [[LZO_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -465,9 +465,9 @@ version = "1.2.1" [[Latexify]] deps = ["Formatting", "InteractiveUtils", "LaTeXStrings", "MacroTools", "Markdown", "Printf", "Requires"] -git-tree-sha1 = "7c72983c6daf61393ee8a0b29a419c709a06cede" +git-tree-sha1 = "f77a16cb3804f4a74f57e5272a6a4a9a628577cb" uuid = "23fbe1c1-3f47-55db-b15f-69d7ec21a316" -version = "0.14.12" +version = "0.15.5" [[LatinHypercubeSampling]] deps = ["Random", "StableRNGs", "StatsBase", "Test"] @@ -568,10 +568,10 @@ uuid = "98b081ad-f1c9-55d3-8b20-4c87d4299306" version = "2.8.1" [[LogExpFunctions]] -deps = ["DocStringExtensions"] -git-tree-sha1 = "9809b844f0ff853f0620e0cac7a712e1818671e5" +deps = ["DocStringExtensions", "LinearAlgebra"] +git-tree-sha1 = "ed26854d7c2c867d143f0e07c198fc9e8b721d10" uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" -version = "0.2.1" +version = "0.2.3" [[Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" @@ -584,39 +584,39 @@ version = "0.6.2" [[MLJ]] deps = ["CategoricalArrays", "ComputationalResources", "Distributed", "Distributions", "LinearAlgebra", "MLJBase", "MLJIteration", "MLJModels", "MLJOpenML", "MLJScientificTypes", "MLJSerialization", "MLJTuning", "Pkg", "ProgressMeter", "Random", "Statistics", "StatsBase", "Tables"] -git-tree-sha1 = "ecf8ec841a9d6aba6257f449fe8f7bfced50f3f0" +git-tree-sha1 = "d629a1e8aa6028ad2dbc1fc23306df4418f09e4a" uuid = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7" -version = "0.16.1" +version = "0.16.4" [[MLJBase]] deps = ["CategoricalArrays", "ComputationalResources", "Dates", "DelimitedFiles", "Distributed", "Distributions", "InteractiveUtils", "InvertedIndices", "LinearAlgebra", "LossFunctions", "MLJModelInterface", "MLJScientificTypes", "Missings", "OrderedCollections", "Parameters", "PrettyTables", "ProgressMeter", "Random", "StatisticalTraits", "Statistics", "StatsBase", "Tables"] -git-tree-sha1 = "92fefe91b67bbffd83d232a85ee86604be356cf7" +git-tree-sha1 = "9f757518de8f8b89defa1f9db31b757d914fe5ac" uuid = "a7f614a8-145f-11e9-1d2a-a57a1082229d" -version = "0.18.1" +version = "0.18.6" [[MLJFlux]] deps = ["CategoricalArrays", "ColorTypes", "ComputationalResources", "Flux", "LossFunctions", "MLJModelInterface", "ProgressMeter", "Statistics", "Tables"] -git-tree-sha1 = "4abb19fe5c1e6bd1a218262eb67dd6b6025536b5" +git-tree-sha1 = "5b769fe228ecf936f733528b346aec7c225ec933" uuid = "094fc8d1-fd35-5302-93ea-dabda2abf845" -version = "0.1.9" +version = "0.1.11" [[MLJIteration]] deps = ["IterationControl", "MLJBase", "Random"] -git-tree-sha1 = "b0f05562d85bb7403e86aaed3d173b39f0d5a747" +git-tree-sha1 = "1649b3156f3a22ef2066c683dbfb3ace6ae9595e" uuid = "614be32b-d00c-4edb-bd02-1eb411ab5e55" -version = "0.2.2" +version = "0.3.0" [[MLJModelInterface]] deps = ["Random", "ScientificTypes", "StatisticalTraits"] -git-tree-sha1 = "96dedd0ca1b75624ff180b265257f3c168047cda" +git-tree-sha1 = "cafa0e923ce1ae659a4b4cb8eb03c98b916f0d4d" uuid = "e80e1ace-859a-464e-9ed9-23947d8ae3ea" -version = "0.4.1" +version = "1.1.0" [[MLJModels]] deps = ["CategoricalArrays", "Dates", "Distances", "Distributions", "InteractiveUtils", "LinearAlgebra", "MLJBase", "MLJModelInterface", "MLJScientificTypes", "OrderedCollections", "Parameters", "Pkg", "REPL", "Random", "Requires", "Statistics", "StatsBase", "Tables"] -git-tree-sha1 = "9d6467dadd07b38ca2cfb0c7e4b6ac0e38372d61" +git-tree-sha1 = "f27b115f55f8e275ed155b858fe57eaf25432afd" uuid = "d491faf4-2d78-11e9-2867-c94bc002c0b7" -version = "0.14.2" +version = "0.14.6" [[MLJOpenML]] deps = ["HTTP", "JSON"] @@ -626,21 +626,21 @@ version = "1.0.0" [[MLJScientificTypes]] deps = ["CategoricalArrays", "ColorTypes", "Dates", "PersistenceDiagramsBase", "PrettyTables", "ScientificTypes", "StatisticalTraits", "Tables"] -git-tree-sha1 = "609b46aca0f1932ab8653464e4194f185f05a864" +git-tree-sha1 = "1df86148d552ed191a1d6f337ae81cf53280f1d7" uuid = "2e2323e0-db8b-457b-ae0d-bdfb3bc63afd" -version = "0.4.4" +version = "0.4.7" [[MLJSerialization]] deps = ["IterationControl", "JLSO", "MLJBase", "MLJModelInterface"] -git-tree-sha1 = "6b962572c761b013a569f1c3436a796ccab33693" +git-tree-sha1 = "cd6285f95948fe1047b7d6fd346c172e247c1188" uuid = "17bed46d-0ab5-4cd4-b792-a5c4b8547c6d" -version = "1.1.0" +version = "1.1.2" [[MLJTuning]] deps = ["ComputationalResources", "Distributed", "Distributions", "LatinHypercubeSampling", "MLJBase", "MLJModelInterface", "ProgressMeter", "Random", "RecipesBase"] -git-tree-sha1 = "4fc52b7dd9c8f6d3a98e686ffb7b9d553f8b2de7" +git-tree-sha1 = "f8d59a74bcbfe3f9753fad13d02aec6c68ac36a4" uuid = "03970b2e-30c4-11ea-3135-d1576263f10f" -version = "0.6.4" +version = "0.6.5" [[MacroTools]] deps = ["Markdown", "Random"] @@ -687,9 +687,9 @@ version = "0.4.4" [[Missings]] deps = ["DataAPI"] -git-tree-sha1 = "f8c673ccc215eb50fcadb285f522420e29e69e1c" +git-tree-sha1 = "4ea90bd5d3985ae1f9a908bd4500ae88921c5ce7" uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" -version = "0.4.5" +version = "1.0.0" [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" @@ -705,9 +705,15 @@ uuid = "14a3606d-f60d-562e-9121-12d972cd8159" [[NNlib]] deps = ["Adapt", "ChainRulesCore", "Compat", "LinearAlgebra", "Pkg", "Requires", "Statistics"] -git-tree-sha1 = "723c0d5252bf95808f934b2384519dd325869f40" +git-tree-sha1 = "80b8360670f445d88b3475e88b33bbcc92f7866e" uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" -version = "0.7.18" +version = "0.7.19" + +[[NNlibCUDA]] +deps = ["CUDA", "LinearAlgebra", "NNlib", "Random", "Statistics"] +git-tree-sha1 = "ecf422ac8bcf33156fb77bf35a02c14e3fd6af18" +uuid = "a00861dc-f156-4864-bf3c-e6376f28a68d" +version = "0.1.1" [[NaNMath]] git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb" @@ -731,9 +737,9 @@ version = "1.1.1+6" [[OpenSpecFun_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "9db77584158d0ab52307f8c04f8e7c08ca76b5b3" +git-tree-sha1 = "b9b8b8ed236998f91143938a760c2112dceeb2b4" uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" -version = "0.5.3+4" +version = "0.5.4+0" [[Opus_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -742,9 +748,9 @@ uuid = "91d4177d-7536-5919-b921-800302f37372" version = "1.3.1+3" [[OrderedCollections]] -git-tree-sha1 = "4fa2ba51070ec13fcc7517db714445b4ab986bdf" +git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.4.0" +version = "1.4.1" [[PCRE_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -800,21 +806,21 @@ version = "1.0.10" [[Plots]] deps = ["Base64", "Contour", "Dates", "FFMPEG", "FixedPointNumbers", "GR", "GeometryBasics", "JSON", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "PlotThemes", "PlotUtils", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs"] -git-tree-sha1 = "cc4eb1be2576984d7a0f7f51478827dee816138b" +git-tree-sha1 = "2628e5859819173cef995470af83db42bf411ef8" uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" -version = "1.11.2" +version = "1.14.0" [[Preferences]] deps = ["TOML"] -git-tree-sha1 = "ea79e4c9077208cd3bc5d29631a26bc0cff78902" +git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.2.1" +version = "1.2.2" [[PrettyTables]] deps = ["Crayons", "Formatting", "Markdown", "Reexport", "Tables"] -git-tree-sha1 = "574a6b3ea95f04e8757c0280bb9c29f1a5e35138" +git-tree-sha1 = "b60494adf99652d220cdef46f8a32232182cc22d" uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" -version = "0.11.1" +version = "1.0.1" [[Printf]] deps = ["Unicode"] @@ -826,9 +832,9 @@ uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" [[ProgressMeter]] deps = ["Distributed", "Printf"] -git-tree-sha1 = "6e9c89cba09f6ef134b00e10625590746ba1e036" +git-tree-sha1 = "1be8800271c86f572d334fef6e3b8364eaece7d9" uuid = "92933f4c-e287-5a05-a399-4b506db050ca" -version = "1.5.0" +version = "1.6.2" [[PyCall]] deps = ["Conda", "Dates", "Libdl", "LinearAlgebra", "MacroTools", "Serialization", "VersionParsing"] @@ -862,6 +868,18 @@ uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" deps = ["Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +[[Random123]] +deps = ["Libdl", "Random", "RandomNumbers"] +git-tree-sha1 = "7c6710c8198fd4444b5eb6a3840b7d47bd3593c5" +uuid = "74087812-796a-5b5d-8853-05524746bad3" +version = "1.3.1" + +[[RandomNumbers]] +deps = ["Random", "Requires"] +git-tree-sha1 = "441e6fc35597524ada7f85e13df1f4e10137d16f" +uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" +version = "1.4.0" + [[RecipesBase]] git-tree-sha1 = "b3fb709f3c97bfc6e948be68beeecb55a0b340ae" uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" @@ -900,9 +918,9 @@ version = "0.3.0+0" uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" [[ScientificTypes]] -git-tree-sha1 = "1d3f5f8bdf5dd0c9951eb9c595ee08a728aec331" +git-tree-sha1 = "b4e89a674804025c4a5843e35e562910485690c2" uuid = "321657f4-b219-11e9-178b-2701a2544e81" -version = "1.1.1" +version = "1.1.2" [[Scratch]] deps = ["Dates"] @@ -919,28 +937,28 @@ uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" [[Showoff]] deps = ["Dates", "Grisu"] -git-tree-sha1 = "236dd0ddad6e3764cce8d8b09c0bbba6df2e194f" +git-tree-sha1 = "91eddf657aca81df9ae6ceb20b959ae5653ad1de" uuid = "992d4aef-0814-514b-bc4d-f2e9a6c4116f" -version = "1.0.2" +version = "1.0.3" [[Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" [[SortingAlgorithms]] -deps = ["DataStructures", "Random", "Test"] -git-tree-sha1 = "03f5898c9959f8115e30bc7226ada7d0df554ddd" +deps = ["DataStructures"] +git-tree-sha1 = "2ec1962eba973f383239da22e75218565c390a96" uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" -version = "0.3.1" +version = "1.0.0" [[SparseArrays]] deps = ["LinearAlgebra", "Random"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[SpecialFunctions]] -deps = ["ChainRulesCore", "OpenSpecFun_jll"] -git-tree-sha1 = "5919936c0e92cff40e57d0ddf0ceb667d42e5902" +deps = ["ChainRulesCore", "LogExpFunctions", "OpenSpecFun_jll"] +git-tree-sha1 = "9146da51b38e9705b9f5ccfadc3ab10a482cae36" uuid = "276daf66-3868-5448-9aa4-cd146d93841b" -version = "1.3.0" +version = "1.4.0" [[StableRNGs]] deps = ["Random", "Test"] @@ -950,25 +968,30 @@ version = "1.0.0" [[StaticArrays]] deps = ["LinearAlgebra", "Random", "Statistics"] -git-tree-sha1 = "2f01a51c23eed210ff4a1be102c4cc8236b66e5b" +git-tree-sha1 = "c635017268fd51ed944ec429bcc4ad010bcea900" uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "1.1.0" +version = "1.2.0" [[StatisticalTraits]] deps = ["ScientificTypes"] -git-tree-sha1 = "0daf443864a1fbb415d782c1dfd161d954140574" +git-tree-sha1 = "2d882a163c295d5d754e4102d92f4dda5a1f906b" uuid = "64bff920-2084-43da-a3e6-9bb72801c0c9" -version = "0.1.1" +version = "1.1.0" [[Statistics]] deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +[[StatsAPI]] +git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.0.0" + [[StatsBase]] -deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics"] -git-tree-sha1 = "4bc58880426274277a066de306ef19ecc22a6863" +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "2f6792d523d7448bbe2fec99eca9218f06cc746d" uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" -version = "0.33.5" +version = "0.33.8" [[StatsFuns]] deps = ["LogExpFunctions", "Rmath", "SpecialFunctions"] @@ -984,9 +1007,9 @@ version = "0.5.1" [[StructTypes]] deps = ["Dates", "UUIDs"] -git-tree-sha1 = "ad4558dee74c5d26ab0d0324766b1a3ee6ae777a" +git-tree-sha1 = "e36adc471280e8b346ea24c5c87ba0571204be7a" uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" -version = "1.7.1" +version = "1.7.2" [[SuiteSparse]] deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] @@ -1023,16 +1046,16 @@ deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [[TimeZones]] -deps = ["Dates", "EzXML", "Mocking", "Pkg", "Printf", "RecipesBase", "Serialization", "Unicode"] -git-tree-sha1 = "4ba8a9579a243400db412b50300cd61d7447e583" +deps = ["Dates", "EzXML", "LazyArtifacts", "Mocking", "Pkg", "Printf", "RecipesBase", "Serialization", "Unicode"] +git-tree-sha1 = "960099aed321e05ac649c90d583d59c9309faee1" uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53" -version = "1.5.3" +version = "1.5.5" [[TimerOutputs]] -deps = ["Printf"] -git-tree-sha1 = "32cdbe6cd2d214c25a0b88f985c9e0092877c236" +deps = ["ExprTools", "Printf"] +git-tree-sha1 = "bf8aacc899a1bd16522d0350e1e2310510d77236" uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" -version = "0.5.8" +version = "0.5.9" [[TranscodingStreams]] deps = ["Random", "Test"] @@ -1041,9 +1064,9 @@ uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" version = "0.9.5" [[URIs]] -git-tree-sha1 = "7855809b88d7b16e9b029afd17880930626f54a2" +git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355" uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" -version = "1.2.0" +version = "1.3.0" [[UUIDs]] deps = ["Random", "SHA"] @@ -1230,9 +1253,9 @@ version = "1.4.8+0" [[Zygote]] deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] -git-tree-sha1 = "de86b4c5ff8e161c37bde0b5ecf6d201721373f8" +git-tree-sha1 = "927209c83efa62256788a9880c191774c07c5b51" uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" -version = "0.6.9" +version = "0.6.10" [[ZygoteRules]] deps = ["MacroTools"] diff --git a/examples/mnist/Project.toml b/examples/mnist/Project.toml index c2d7ca74..332b068f 100644 --- a/examples/mnist/Project.toml +++ b/examples/mnist/Project.toml @@ -1,5 +1,4 @@ [deps] -EarlyStopping = "792122b4-ca99-40de-a6bc-6742525f08b6" Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" MLJ = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7" diff --git a/examples/mnist/mnist.ipynb b/examples/mnist/mnist.ipynb index c6e70d66..3dc1ce2d 100644 --- a/examples/mnist/mnist.ipynb +++ b/examples/mnist/mnist.ipynb @@ -2,46 +2,34 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, "source": [ "# Using MLJ to classifiy the MNIST image dataset" - ] + ], + "metadata": {} }, { + "outputs": [], "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[32m\u001b[1m Activating\u001b[22m\u001b[39m environment at `~/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/Project.toml`\n", - "\u001b[32m\u001b[1mPrecompiling\u001b[22m\u001b[39m project...\n", - "\u001b[32m ✓ \u001b[39m\u001b[90mStatsFuns\u001b[39m\n", - "\u001b[32m ✓ \u001b[39m\u001b[90mDistributions\u001b[39m\n", - "\u001b[32m ✓ \u001b[39m\u001b[90mMLJBase\u001b[39m\n", - "\u001b[32m ✓ \u001b[39m\u001b[90mMLJIteration\u001b[39m\n", - "\u001b[32m ✓ \u001b[39m\u001b[90mMLJTuning\u001b[39m\n", - "\u001b[32m ✓ \u001b[39m\u001b[90mMLJSerialization\u001b[39m\n", - "\u001b[32m ✓ \u001b[39m\u001b[90mMLJModels\u001b[39m\n", - "\u001b[32m ✓ \u001b[39mMLJ\n", - "8 dependencies successfully precompiled in 21 seconds (188 already precompiled)\n", - "┌ Info: Precompiling Flux [587475ba-b771-5e3f-ad9e-33799f191a9c]\n", - "└ @ Base loading.jl:1317\n", - "┌ Info: Precompiling MLJFlux [094fc8d1-fd35-5302-93ea-dabda2abf845]\n", - "└ @ Base loading.jl:1317\n", - "┌ Info: Precompiling Plots [91a5bcdd-55d7-5caf-9e0b-520d859cae80]\n", - "└ @ Base loading.jl:1317\n" - ] - } - ], "source": [ "using Pkg\n", "const DIR = @__DIR__\n", "Pkg.activate(DIR)\n", - "Pkg.instantiate()\n", - "\n", + "Pkg.instantiate()" + ], + "metadata": {}, + "execution_count": null + }, + { + "cell_type": "markdown", + "source": [ + "**Julia version** is assumed to be 1.6.*" + ], + "metadata": {} + }, + { + "outputs": [], + "cell_type": "code", + "source": [ "using MLJ\n", "using Flux\n", "import MLJFlux\n", @@ -52,928 +40,98 @@ "\n", "using Plots\n", "pyplot(size=(600, 300*(sqrt(5)-1)));" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "## Basic training" - ] + ], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "Downloading the MNIST image dataset:" - ] + ], + "metadata": {} }, { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, "outputs": [], + "cell_type": "code", "source": [ "import Flux.Data.MNIST\n", "images, labels = MNIST.images(), MNIST.labels();" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "In MLJ, integers cannot be used for encoding categorical data, so we\n", "must force the labels to have the `Multiclass` [scientific\n", "type](https://alan-turing-institute.github.io/MLJScientificTypes.jl/dev/). For\n", "more on this, see [Working with Categorical\n", "Data](https://alan-turing-institute.github.io/MLJ.jl/dev/working_with_categorical_data/)." - ] + ], + "metadata": {} }, { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, "outputs": [], + "cell_type": "code", "source": [ "labels = coerce(labels, Multiclass);" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "Checking scientific types:" - ] + ], + "metadata": {} }, { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, "outputs": [], + "cell_type": "code", "source": [ "@assert scitype(images) <: AbstractVector{<:Image}\n", "@assert scitype(labels) <: AbstractVector{<:Finite}" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "Looks good." - ] + ], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "For general instructions on coercing image data, see [Type coercion\n", "for image\n", "data](https://alan-turing-institute.github.io/MLJScientificTypes.jl/dev/#Type-coercion-for-image-data-1)" - ] + ], + "metadata": {} }, { + "outputs": [], "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "28×28 Array{Gray{N0f8},2} with eltype Gray{FixedPointNumbers.N0f8}:\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) … Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) … Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) … Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " ⋮ ⋱ \n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) … Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) … Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)\n", - " Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0) Gray{N0f8}(0.0)" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "images[1]" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "We start by defining a suitable `Builder` object. This is a recipe\n", "for building the neural network. Our builder will work for images of\n", @@ -982,13 +140,12 @@ "alternating convolution and max-pool layers, and a final dense\n", "layer; the filter size and the number of channels after each\n", "convolution layer is customisable." - ] + ], + "metadata": {} }, { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, "outputs": [], + "cell_type": "code", "source": [ "import MLJFlux\n", "struct MyConvBuilder\n", @@ -1022,313 +179,160 @@ " flatten,\n", " Dense(h*w*c3, n_out))\n", "end" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "**Note.** There is no final `softmax` here, as this is applied by\n", "default in all MLJFLux classifiers. Customisation of this behaviour\n", "is controlled using using the `finaliser` hyperparameter of the\n", "classifier." - ] + ], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "We now define the MLJ model. If you have a GPU, substitute\n", "`acceleration=CUDALibs()` below:" - ] + ], + "metadata": {} }, { + "outputs": [], "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "┌ Info: For silent loading, specify `verbosity=0`. \n", - "└ @ Main /Users/anthony/.julia/packages/MLJModels/zYlo3/src/loading.jl:168\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "import MLJFlux ✔\n" - ] - }, - { - "data": { - "text/plain": [ - "ImageClassifier(\n", - " builder = MyConvBuilder(3, 16, 32, 32),\n", - " finaliser = NNlib.softmax,\n", - " optimiser = ADAM(0.001, (0.9, 0.999), IdDict{Any, Any}()),\n", - " loss = Flux.Losses.crossentropy,\n", - " epochs = 10,\n", - " batch_size = 50,\n", - " lambda = 0.0,\n", - " alpha = 0.0,\n", - " optimiser_changes_trigger_retraining = false,\n", - " acceleration = CPU1{Nothing}(nothing)) @839" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "ImageClassifier = @load ImageClassifier\n", "clf = ImageClassifier(builder=MyConvBuilder(3, 16, 32, 32),\n", " acceleration=CPU1(),\n", " batch_size=50,\n", " epochs=10)" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "You can add Flux options `optimiser=...` and `loss=...` here. At\n", "present, `loss` must be a Flux-compatible loss, not an MLJ measure." - ] + ], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "Binding the model with data in an MLJ machine:" - ] + ], + "metadata": {} }, { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, "outputs": [], + "cell_type": "code", "source": [ "mach = machine(clf, images, labels);" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "Training for 10 epochs on the first 500 images:" - ] + ], + "metadata": {} }, { + "outputs": [], "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "┌ Info: Training Machine{ImageClassifier{MyConvBuilder,…},…} @110.\n", - "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/KWyqX/src/machines.jl:342\n", - "┌ Info: Loss is 2.239\n", - "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/AeMUx/src/core.jl:143\n", - "┌ Info: Loss is 2.109\n", - "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/AeMUx/src/core.jl:143\n", - "┌ Info: Loss is 1.814\n", - "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/AeMUx/src/core.jl:143\n", - "┌ Info: Loss is 1.269\n", - "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/AeMUx/src/core.jl:143\n", - "┌ Info: Loss is 0.7602\n", - "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/AeMUx/src/core.jl:143\n", - "┌ Info: Loss is 0.5445\n", - "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/AeMUx/src/core.jl:143\n", - "┌ Info: Loss is 0.4606\n", - "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/AeMUx/src/core.jl:143\n", - "┌ Info: Loss is 0.341\n", - "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/AeMUx/src/core.jl:143\n", - "┌ Info: Loss is 0.2975\n", - "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/AeMUx/src/core.jl:143\n", - "┌ Info: Loss is 0.258\n", - "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/AeMUx/src/core.jl:143\n" - ] - } - ], "source": [ "fit!(mach, rows=1:500, verbosity=2);" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "Inspecting:" - ] + ], + "metadata": {} }, { + "outputs": [], "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(training_losses = Float32[2.3228688, 2.2390091, 2.1091332, 1.8143247, 1.2688795, 0.76020443, 0.54449147, 0.46060592, 0.34104383, 0.2975061, 0.25796312],)" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "report(mach)" - ] + ], + "metadata": {}, + "execution_count": null }, { + "outputs": [], "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(chain = Chain(Chain(Conv((3, 3), 1=>16, relu), MaxPool((2, 2), pad = (0, 0, 0, 0), stride = (2, 2)), Conv((3, 3), 16=>32, relu), MaxPool((2, 2), pad = (0, 0, 0, 0), stride = (2, 2)), Conv((3, 3), 32=>32, relu), MaxPool((2, 2), pad = (0, 0, 0, 0), stride = (2, 2)), flatten, Dense(288, 10)), softmax),)" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "chain = fitted_params(mach)" - ] + ], + "metadata": {}, + "execution_count": null }, { + "outputs": [], "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "16-element Vector{Float32}:\n", - " 0.009390121\n", - " 0.07259897\n", - " -0.0038282399\n", - " 0.016712524\n", - " 0.001980654\n", - " 0.027747674\n", - " -0.0007374671\n", - " 0.00018301565\n", - " 0.07081605\n", - " 0.06926995\n", - " 0.0020753616\n", - " 0.0032082414\n", - " 0.015448393\n", - " 0.008061441\n", - " 0.023986094\n", - " 0.04710653" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "Flux.params(chain)[2]" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "Adding 20 more epochs:" - ] + ], + "metadata": {} }, { + "outputs": [], "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "┌ Info: Updating Machine{ImageClassifier{MyConvBuilder,…},…} @110.\n", - "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/KWyqX/src/machines.jl:343\n", - "\u001b[33mOptimising neural net:100%[=========================] Time: 0:00:07\u001b[39m\n" - ] - } - ], "source": [ "clf.epochs = clf.epochs + 20\n", "fit!(mach, rows=1:500);" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "Computing an out-of-sample estimate of the loss:" - ] + ], + "metadata": {} }, { + "outputs": [], "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.36543968f0" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "predicted_labels = predict(mach, rows=501:1000);\n", "cross_entropy(predicted_labels, labels[501:1000]) |> mean" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "Or, in one line (after resetting the RNG seed to ensure the same\n", "result):" - ] + ], + "metadata": {} }, { + "outputs": [], "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "┌───────────────────────┬───────────────┬────────────────┐\n", - "│\u001b[22m _.measure \u001b[0m│\u001b[22m _.measurement \u001b[0m│\u001b[22m _.per_fold \u001b[0m│\n", - "├───────────────────────┼───────────────┼────────────────┤\n", - "│ LogLoss{Float64} @358 │ 0.365 │ Float32[0.365] │\n", - "└───────────────────────┴───────────────┴────────────────┘\n", - "_.per_observation = [[[6.12, 0.182, ..., 0.00043]]]\n", - "_.fitted_params_per_fold = [ … ]\n", - "_.report_per_fold = [ … ]\n" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "Random.seed!(123)\n", "evaluate!(mach,\n", @@ -1336,127 +340,91 @@ " measure=cross_entropy,\n", " rows=1:1000,\n", " verbosity=0)" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "## Wrapping the MLJFlux model with iteration controls" - ] + ], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "Any iterative MLJ model implementing the warm restart functionality\n", "illustrated above for `ImageClassifier` can be wrapped in *iteration\n", "controls*, as we demonstrate next. For more on MLJ's\n", "`IteratedModel` wrapper, see the [MLJ\n", "documentation](https://alan-turing-institute.github.io/MLJ.jl/dev/controlling_iterative_models/)." - ] + ], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "The \"self-iterating\" model, called `imodel` below, is for iterating the\n", "image classifier defined above until one of the following stopping\n", "criterion apply:" - ] + ], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "- `Patience(3)` (3 consecutive increases in the loss)" - ] + ], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "- `InvalidValue()` (an out-of-sample loss, or a training loss,\n", " is `NaN`, `Inf`, or `-Inf`)" - ] + ], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "- `TimeLimit(t=1/60)` (training time has exceeded one minute)" - ] + ], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "Additionally, training a machine bound to `imodel` will:" - ] + ], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "- save a snapshot of the machine every three epochs" - ] + ], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "- record the out-of-sample loss and training losses for plotting" - ] + ], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "For a complete list of controls, see [this\n", "table](https://alan-turing-institute.github.io/MLJ.jl/dev/controlling_iterative_models/#Controls-provided)." - ] + ], + "metadata": {} }, { + "outputs": [], "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "ProbabilisticIteratedModel(\n", - " model = ImageClassifier(\n", - " builder = MyConvBuilder(3, 16, 32, 32),\n", - " finaliser = NNlib.softmax,\n", - " optimiser = ADAM(0.001, (0.9, 0.999), IdDict{Any, Any}()),\n", - " loss = Flux.Losses.crossentropy,\n", - " epochs = 30,\n", - " batch_size = 50,\n", - " lambda = 0.0,\n", - " alpha = 0.0,\n", - " optimiser_changes_trigger_retraining = false,\n", - " acceleration = CPU1{Nothing}(nothing)),\n", - " controls = Any[Step(1), Patience(2), InvalidValue(), TimeLimit(Dates.Millisecond(1800000)), Save{Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}}(\"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine.jlso\", Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}()), WithLossDo{IterationControl.var\"#16#18\"}(IterationControl.var\"#16#18\"(), false, nothing), WithLossDo{typeof(add_loss)}(add_loss, false, nothing), WithTrainingLossesDo{typeof(add_training_loss)}(add_training_loss, false, nothing)],\n", - " resampling = Holdout(\n", - " fraction_train = 0.7,\n", - " shuffle = false,\n", - " rng = Random._GLOBAL_RNG()),\n", - " measure = LogLoss(\n", - " tol = 2.220446049250313e-16),\n", - " weights = nothing,\n", - " class_weights = nothing,\n", - " operation = MLJModelInterface.predict,\n", - " retrain = false,\n", - " check_measure = true,\n", - " iteration_parameter = nothing,\n", - " cache = true) @561" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "losses = []\n", "training_losses = [];\n", @@ -1476,271 +444,102 @@ " resampling=Holdout(fraction_train=0.7),\n", " measure=log_loss,\n", " retrain=false)" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "Binding our self-iterating model to data:" - ] + ], + "metadata": {} }, { + "outputs": [], "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Machine{ProbabilisticIteratedModel{ImageClassifier{MyConvBuilder,…}},…} @130 trained 0 times; does not cache data\n", - " args: \n", - " 1:\tSource @566 ⏎ `AbstractVector{GrayImage{28, 28}}`\n", - " 2:\tSource @832 ⏎ `AbstractVector{Multiclass{10}}`\n" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "mach = machine(imodel, images, labels)" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "And training on the first 500 images:" - ] + ], + "metadata": {} }, { + "outputs": [], "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "┌ Info: Training Machine{ProbabilisticIteratedModel{ImageClassifier{MyConvBuilder,…}},…} @130.\n", - "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/KWyqX/src/machines.jl:342\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine1.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 2.2725065\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine2.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 2.212589\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine3.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 2.109312\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine4.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 1.9105355\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine5.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 1.5846978\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine6.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 1.1751547\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine7.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 0.84284645\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine8.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 0.65616626\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine9.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 0.5708759\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine10.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 0.5298943\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine11.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 0.50618887\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine12.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 0.483175\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine13.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 0.46368352\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine14.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 0.46107793\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine15.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 0.45514902\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine16.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 0.4517515\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine17.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 0.45229506\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine18.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 0.4490886\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine19.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 0.44740736\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine20.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 0.44860032\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: Saving \"/Users/anthony/Dropbox/Julia7/MLJ/MLJFlux/examples/mnist/mnist_machine21.jlso\". \n", - "└ @ MLJSerialization /Users/anthony/.julia/packages/MLJSerialization/UX4yW/src/controls.jl:33\n", - "┌ Info: loss: 0.451104\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/controls.jl:280\n", - "┌ Info: final loss: 0.451104\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/train.jl:27\n", - "┌ Info: final training loss: 0.06117626\n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/train.jl:29\n", - "┌ Info: Stop triggered by Patience(2) stopping criterion. \n", - "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/TOn4C/src/stopping_controls.jl:75\n", - "┌ Info: Total of 21 iterations. \n", - "└ @ MLJIteration /Users/anthony/.julia/packages/MLJIteration/PKvIw/src/core.jl:35\n" - ] - }, - { - "data": { - "text/plain": [ - "Machine{ProbabilisticIteratedModel{ImageClassifier{MyConvBuilder,…}},…} @130 trained 1 time; does not cache data\n", - " args: \n", - " 1:\tSource @566 ⏎ `AbstractVector{GrayImage{28, 28}}`\n", - " 2:\tSource @832 ⏎ `AbstractVector{Multiclass{10}}`\n" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "fit!(mach, rows=1:500)" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "A comparison of the training and out-of-sample losses:" - ] + ], + "metadata": {} }, { + "outputs": [], "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAFyCAYAAAApuaQRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3gVxeLG8e/uOemVFDoh9CIlBelNQJCm0lVQ+YmACmLn2hVEsXCtgKJXRcVyFSxIE0Gko1QRQ5UuCEkgIY0k55z9/RGJ5qbQ0vN+nocHMzO7OzlZzevs7IxhWZaFiIiIiBQas6Q7ICIiIlLeKGCJiIiIFDIFLBEREZFCpoAlIiIiUsgUsETKsc2bNzNq1CgaNGiAj48PXl5e1KtXj5tvvpnvv/++pLtXKGbPno1hGAX+uffeey/rGoZh0LVr18LpsIhUCPaS7oCIFD6Xy8WDDz7IK6+8gt1up1u3blx77bW4ubmxf/9+Fi5cyJw5c5g8eTJPPPFESXe3UHTv3p2OHTvmWde2bdti7o2IVHQKWCLl0OOPP84rr7xCREQEc+fOpV69ejnq09LSmD59OvHx8SXUw8LXo0cPHn744ZLuhogIoEeEIuXOvn37ePHFFwkODmbJkiW5whWAl5cXDz30EJMmTcouGzlyJIZhsH//fl555RWuuOIKPDw8GDlyZHab3377jWHDhlG5cmU8PDyoU6cO9913H6dOncp1jb179/J///d/1KlTB09PT0JCQoiKiuKBBx7I0e748ePcc889NGjQAC8vL4KCgmjevDl33XUXZ86cKbwP5i8//vgjhmHw9NNPs2XLFnr16oWfnx8BAQEMGDCAgwcP5moLsHLlyhyPHWfPng38/Yhy9uzZLFy4kE6dOuHn50d4eHj2eeLj47nvvvuoU6cOHh4eVK5cmWHDhhETE5Orf+d+Dr///jtTp06lfv36eHp60qBBA1566SVcLld22xUrVmAYBuPGjcvze42JicEwDK677rrL/+BE5KJoBEuknJk9ezZOp5OxY8dSpUqVAtt6eHjkKrv77rvZsGEDffv2pV+/ftnnWLduHT179iQ9PZ3BgwcTHh7Ohg0bePXVV1m4cCHr168nODgYgGPHjtG6dWtSUlLo27cvw4YNIzk5mb179/LGG2/w73//G4DU1FQ6dOjAwYMH6dmzJwMGDCAjI4P9+/cze/ZsJk6ciL+/fyF/Qlk2bdrESy+9RNeuXRk7dixbt27l66+/5tdff2XHjh14enoSHh7OU089xaRJk6hdu3aOsBkREZHjfF988QVLly6lX79+3HXXXSQlJQFZ4apt27bs27ePrl27csMNN3Dw4EHmzp3LwoUL+f7772nXrl2u/t17771s2LCBoUOH4unpyZdffsnEiRPZt28fs2bNAuCqq66iYcOGfPzxx0ybNg0vL68c5/jPf/4DwOjRowvzoxORC2GJSLnStWtXC7CWLVt2UcfdeuutFmDVrFnTOnToUI46p9NpNWjQwAKsJUuW5Kh75JFHLMAaNWpUdtnrr79uAdZrr72W6zqxsbHZ/zx//nwLsO67775c7c6cOWOlp6eft9/vv/++BVjdu3e3nnrqqTz/7Ny5M7v9ihUrLMACrM8++yzHuW6++WYLsD799NMc5YDVpUuXAq9vGIb1/fff56q/7bbbLMB65JFHcpQvWbLEAqwGDRpYTqczu/zcz6FKlSrWH3/8kV2elJRkNW/e3AKsVatWZZe/9NJLFmB98MEHOc6fnp5uhYSEWDVq1LAcDkc+n56IFBUFLJFypnHjxhZg7dq166KOO/eLPa9QtGrVKguwevfunasuOTnZCg4Otry8vLID0bmA9fbbbxd4zXMB69FHH72ovv7TuYBT0J+vvvoqu/25gNW5c+dc5zpXd//99+cov5CANWDAgFx16enplpeXlxUcHGylpKTkqu/Vq5cFWKtXr84uO/dzePbZZ3O1/+KLL3KF2djYWMvDw8Pq1KlTjraff/65BViPP/54nv0WkaKlOVgikkPr1q1zlW3duhUgz6UKfHx8aNWqFWlpaezZsweAfv364e3tzbhx4xg6dCjvvfdedt0/de7cmapVqzJ16lT69u3LzJkz2b59O9YlbJE6depUrKz/acz15/rrr8/VPioqKldZzZo1AUhISLjo6+f1ue3atYu0tDRat26Nt7d3rvpzn+e2bdty1XXq1Cnfsn+2DwkJYeDAgaxevTrHZ/zuu+9iGAajRo266O9FRC6fApZIOVO1alUA/vjjj0s6Pq95W+cmm+c3p+vcNRMTEwGoU6cO69evp3///ixevJhRo0bRqFEjGjduzBdffJF9XEBAAOvXr+fmm29m/fr1jBs3jpYtWxIWFsbMmTMvqf8XKiAgIFeZ3Z41LdXpdF70+Qrjc/unypUr51lmmmau9mPGjAH+nnN1+PBhvv/+e3r06JFjsr2IFB8FLJFypkOHDgAsX778ko4/99bcP52baH7ixIk8jzlX/s8J6S1atGDevHmcOnWK9evX8+STT3LixAmGDRvG2rVrs9uFh4fzwQcfEBsby9atW3nhhRewLItx48bx6aefXtL3UBIK63M75+TJk3mWuVyuXOGwa9euNGrUiA8//JDMzEzee+89XC6XJreLlCAFLJFyZuTIkdhsNt5++21iY2MLbJuenn5B54yMjASyli34X6mpqWzatAkvLy8aNWqUq97NzY22bdsyadIkXn/9dSzLYsGCBbna2Ww2IiIimDhxYnawmj9//gX1r6iZpnlJo1qNGzfG09OTjRs3kpqamqt+5cqVQO43EgFWr16db1le7UePHs2JEyf45ptveP/99wkJCdHyDCIlSAFLpJypX78+EydOJC4ujt69e3PgwIFcbc6ePcvLL7/M008/fUHn7NChA/Xq1WPx4sUsW7YsR93UqVOJi4vjxhtvxN3dHYCNGzfmOQJzbsTm3HICO3bs4NChQ+dtV9KCgoI4evToRR/n7u7OjTfeSFxcHFOnTs1Rt2zZMhYvXkz9+vWzRx3/6fXXX+fYsWPZXycnJzN58mQAbrnlllztR44ciYeHB/fccw+HDx/m1ltvzf55iEjx0zpYIuXQlClTOHv2LK+88gqNGjWiW7duNGvWDDc3Nw4cOMCyZcuIj49nypQpF3Q+0zSZPXs2vXr1ok+fPgwZMoTatWvz008/8cMPP1CvXj2ef/757PYff/wxM2fOpGvXrtSvXx9/f39iYmJYtGgRISEh3HbbbUBWyHjggQfo0KEDjRs3Jjg4mP379zN//ny8vLwYP378BX/Py5Yt4+zZs3nWhYeH51jD6mJ169aNzz//nMGDBxMZGYnNZqNv3740b978vMe+8MILrFy5kilTprBu3TratGmTvQ6Wt7c377//PqaZ+/91r7zySlq2bMmwYcPw8PDgyy+/5ODBg4wePZrOnTvnah8cHMygQYP45JNPALj99tsv+fsVkUJQYu8vikiR27hxo3XbbbdZ9evXt7y8vCwPDw8rPDzcuvHGG62lS5fmaHtueYADBw7ke77t27dbgwcPtkJCQiw3Nzerdu3a1oQJE3KsbWVZlrVhwwZr7NixVrNmzazAwEDLy8vLatCggTVhwgTr8OHD2e1iYmKse+65x4qMjLSCg4MtDw8Pq27dutbIkSOtmJiYC/oeL2SZhn8usXBuKYannnoq17kOHDhgAdatt96ao/z48ePW0KFDrZCQEMs0TQuw3n///RzXP/d1XmJjY60JEyZYtWvXttzc3KyQkBBr8ODB1q+//pqr7bmfw759+6znnnvOqlu3ruXu7m7Vq1fPeuGFFwpc0+q7776zAKtjx44FfWQiUgwMy7qE96FFRKRIjBw5kg8++IADBw5c9BuAL774Iv/617/44IMP8nyMKCLFR3OwRETKgbNnzzJjxgyCgoIYMmRISXdHpMLTHCwRkTJszZo1rFy5ku+++47Dhw/z/PPPl5qXA0QqMgUsEZEybNmyZUyaNImQkBDuu+8+HnjggZLukogAmoMlIiIiUsg0B0tERESkkBVrwEpNTWXLli15rmgsIiIiUl4Ua8DatWsX0dHR7Nq1qzgvKyUkrw1sRXRfSF50X0heyvJ9oUeEUmQuZe82Kf90X0hedF9IXsryfaGAJSIiIlLIFLBERERECpkCloiIiEgh00KjIiIil+jw4cPExcWVdDfKrcTERAICAkq6G7mEhIQQFhZWYBsFLBERkUtw+PBhmjRpoqWHKiBvb2927txZYMhSwBIREbkEcXFxpKamMmfOHJo0aVLS3ZFisnPnTkaMGEFcXFz5CVixaRav7XDRJ8ygTaiBzTRKuksiIlLBNWnShKioqJLuhpQyZSpg7UqweHOni2e3QZAH9Kxp0KeWyTU1DUK9FLZERESkdChTAatDQCr7a37H7lqdWJAYzKIjFrf87sQArgw16FPLoHctg1ahBqahwCUiIiIlo0wFrIyje0laOJvqzncYX6cpEyO7kNSuI9+dqcTiIy5e2eHi6S0Q6gnX1DToE2bSs4ZBkKfCloiIiBSfMhWwPBtGUn3KZ6T9up60bStJ+PptsN6ib73mDI7sgvuA9vyUEsCiIxaLj7j4aJ8T04C2lc+NbplEBKPRLRERESlSZSpgAZhePvi07oFP6x64UpJI+3UtqVtXkTBvBsybQeMGEURFdubZazpwzPJlyVGLRUdcPP+Li8c3uajqBb1rZc3durqmQYC7wpaIiIgUrjIXsP7J9PHDp+01+LS9BmdyAmm/rCVt2ypO//c1Tn/+Bp6No7gxogu3dWyHw92btScsFh2xWHTYxft7nNgM6FDl79Gt5kFgaHRLRERELlOZDlj/ZPMNxLdDX3w79MV55hRpv6whdetKTn8yjdM2NzybtKJNZGe6RLTlpTZeHErKeoy46IjF5K0uHt7oooYP9PlrdKt7dQM/jW6JiIjIJSiXexHa/IPw7XQtlSf8m6pPf0RA///DlXSaUx+9wLHHhxH//hRCf1/NmPqZzO9lJ/5mO9/1tjGkjsnK4xYDvncS8pGDMasdHEyySvrbERERKTYJCQm8+OKLl3z877//TlRUFJGRkbz//vuF2LNLd/DgQUJCQor1muVmBCs/9sBQ/LoOxK/rQBzxf5K2bTWpW1dyavZzGO6eeF7RBu/ILlzdpBU9a7rzSjsbv5+x+GK/i5d/dfH+bgcjGxo8GmGjjr9GtEREpHw7F7AmTpx4ScfPnTuXdu3aMWPGjELuWdlSLkew8mMPropf9yFUeXA6VR97D78ew3CcOEz8e5M59sQNnJrzEmm//URdbwcPR9g4cIOdqa1N5h+2aPi5g1ErHew/oxEtERG5cMdTLbbEFf6f46kX9vtoyZIlREVF0aJFC7p06UJMTAw//vgjrVq1ym6zY8cOwsPDAbjjjjtISEggIiIiR5t/Sk5O5rbbbqNZs2Y0a9aMSZMmAfDhhx/yyiuv8MUXXxAREUFMTEyO4zZs2EB0dDQRERE0a9aMN998E4BPPvmENm3aEBkZSUREBIsWLco+Jjw8nCeffJL27dsTFhbGnDlzeO2112jdujX16tXjxx9/BP4epXrwwQdp06YNV1xxBT/88EOe/d+4cSPdunWjVatWREVFMW/evAv6LC9GuR/Byo89tDr+PW/Ev+eNZP55mLRtq0jduorUTcsxvH3xatEBv25DeLBFTe5qavJWjIsXt7v4YK+DmxsYPBZho36ARrRERKRgs3a6mLTFVejnfSrK5OloW4FtTp48yYgRI1ixYgXNmzfn448/ZujQoUyfPj3fY9566y1atWrFtm3b8m3zzDPPkJGRwfbt20lLS6Njx440bdqUW265hf3795OcnMy0adNyHTd16lQeeOABbrrpJgBOnz4NQK9evbjxxhsxDIODBw/Svn17Dh06hJubGwBpaWmsW7eOjRs30qVLF6ZNm8bPP//M559/zqOPPsq6desAiI+Pp3nz5kybNo0NGzZw/fXX8/vvv+foQ0JCAmPHjmXhwoVUq1aNuLg4oqOj6dChA1WrVi3w87wYFTZg/ZNb1TDcrhmBX6/hOI4fJHXrSlI3Lid143J8O1+Hf8+buL+FD3c0NXl7p4sXfnHx0V4HI+obPBZpo4GCloiI5GNsE5Nraxf+A6Nq3udv89NPPxEREUHz5s0BGD58OOPGjeP48eOXde1ly5bx2muvYZomPj4+3HLLLSxbtowhQ4YUeNxVV13FlClT2LdvH926daNjx44AHDhwgOHDh3P06FHsdjtxcXEcOnSI+vXrAzBs2DAAoqKiSEtLY+jQoQBER0ezf//+7PO7u7tz8803A9C2bVuqVq3KL7/8QvXq1bPbrFu3jv3799O7d+/sMsuy2L17twJWUTEMA7fqdQioXgf/q28k6ccvSVr2GakblxPQbyTera/m3uY2xjYxeXvXX0Frn4Ph9Qwej7TRMFBBS0REcqrmbVxQGCoKlmXlufxQrVq1cDqd2V+fPXs233PExMRkjzh16NCBGTNm5HnevK4zePBg9u3bB8Dy5cu59957ufbaa1m+fDmPPvoozZo1Y+bMmdxwww1MmzaN66+/HoCgoKAcffL09ATAZrPl+trhcBT4GfxvvyzLokWLFqxatarA4y5XhZqDdTEMdw/8e95I1UffxbNxFKc/e5WTL99D+v4deNkN7mlmY/8wO6+2NVl+zKLJXAcjVjjYlaA5WiIiUjq0a9eObdu2sXPnTgA+++wzatasSZ06dThw4ADx8fEAfPTRR9nH+Pv7k5qamh1cmjZtyrZt29i2bVv2xPWrr76ad955B8uySElJYc6cOfTo0SPX9efOnZt9bHBwMLt376Zu3bqMHj2aRx99lA0bNgBZjwrPzQGbM2dO9qPDi5WRkcHHH38MwM8//8yff/5JixYtcrRp3749e/fuzTE/a9u2bWRkZFzSNfOjEazzsAWGEDRiIj4d+pHw1VvEvv4gXpFdCLh2FJ6VKnN3MxujG5u8u9vF1F9cfLLPwY1/jWg1qaQRLRERKTmhoaF89NFHDB8+HKfTSWBgIJ9//jk1atTgwQcfpFWrVoSHh9O5c+fsY4KCghg+fDjNmzfHx8eHTZs25TrvE088wd1335396HHIkCEMHjz4vP154403WLFiBe7u7thsNv79738D8NprrzFgwABq1KhBu3btCAsLu6TvNzg4mH379tGmTRuSk5P55JNP8PHxITY2NrtNpUqV+Pbbb3nooYe47777yMzMJCwsjK+//vqSrpkfw7KsYhty2bJlC9HR0WzevJmoqKjiumyhsVwuUjctJ3HBe1hpqfh1H4Jvt8GY7llDlelOi/d2u5i6zcXRFBhWz+CJSBtNK2jQOnXqFEFBQSXdDSlldF9IXsrifVHWf6eVNwcPHqRVq1bExcUV6XUu9OeuR4QXwTBNfFpfTdVH38W383Wc+f6/nHhuNKlbVmJZFh42gzub2tg7zM6bHU3WnbBoNtfBsOUOdpzSo0MREZGKQgHrEpie3gT0v42qj8zCrVYDTn04ldg3HiTjyF4APGwGY5vY2DvUzlsdbfx00qL5PAdDlzn4VUFLRESk0IWHhxf56NXFUMC6DPaQ6oSMepKQO5/DlZrMyZcncPqzV3EmJQDgbjMY08Rkz1A773SysTHWosU8B4OXOdger6AlIiJSXilgFQLPRlFUeWgmgQPvJHX7Gv589jaSVszDcmQCWUHr9sYme4bZebezja1xFi2/dDDwewfbFLRERETKHQWsQmLYbPh2upaqj72H95U9SPz2XU68cAdpv/2U3cbNNLitkcmuoXbe72Jj+ymLyC8dDPrewel0BS0REZHyQgGrkNl8/Kk06C6qPDQTW6VQ4t95iti3HifzxJHsNm6mwciGJruG2JndxcaPxy26LXQQm6aQJSIiUh4oYBURt2rhhNw5leDbnsQR+wcnXriDhK9m4UpNzm5jNw1ubWjyYz87x1OhywIHx1IUskREpPA8/fTTl7SI5qZNmxg+fPgFtY2IiCAtLe2ir1GeKWAVIcMw8GrRnqoPz8K/zy2krF/Mn8+OInntQizX31sUNA8yWNXPTlImdF7g4FCSQpaIiBSOSZMm5RmwzrfFTKtWrbJXRT+fbdu24eXldUn9K68UsIqB4eaOf49hVH3sXTybXknCF29wctrdpO/bnt2mYaDB6v52LAs6fetgb6JCloiIXJ477rgDyNoeJiIigj59+jBhwgSuueYaWrZsCcCIESNo1aoVLVq0oF+/fpw8eRKAH3/8kVatWgFZi3iGhITw5JNPEh0dTf369Vm0aFH2dQzDIDk56wlNeHg4kyZNon379tSpU4cpU6Zkt4uJiaFNmzY0a9aMm266ibZt27JgwYJi+SyKm7bKKUa2gGCChj+Ib8f+JHz5JrHTJ+IV0YmA/qOwB1cl3M9gVX87PRY56PStg2V97DQLqpirwIuIlBfOxHicZ04V+nlt/kHYAoILbPPWW28xa9Ys1q1bh6+vLyNHjmTNmjWsWrUKX19fAF599VVCQkIAeP7555k8eTLTp0/Pda74+Hiio6OZPHkyS5Ys4Z577qFPnz55XjchIYF169YRGxtL/fr1+b//+z9q1KjBzTffzH333ceIESPYvHkzrVu3vsxPofRSwCoB7rUbEXrPy6RuWUHit+9x4oWxBI+ejGeDltTwMVjZz07PRQ66LnDwXW870aEKWSIiZVXyukUkfXdhj9ouhl+v4QT0vvmijxs6dGh2uAL4+OOP+eijj0hPTyctLY2qVavmeZyPjw/XXXcdkLWJ9O+//57vNc7N3QoNDaVu3bocOHAAPz8/duzYwU033QRAdHR0ro2YyxMFrBJimCY+rbrj1bw98e89Q/w7TxJyx7N41G1GZS+DFf3s9F7ipNtCB4t722hfRU9zRUTKIt/2ffBq1rbQz2vzv7S9G/8ZrtasWcP06dNZt24doaGhzJ8/n8mTJ+d5nKen59/XttlwOp15tsurrcPhwLIsDMPAMCrGoIECVgkzPbwIGfUUce88SdysrFXhPcIbU8nD4PveNvp956TnIifze0K3GgpZIiJljS0g+LyP8oqSn58fiYmJOYLVOadPn8bf35+goCAyMjKYNWtWkfUjICCApk2b8umnn3LTTTexdetWfv311yK7XknTb+xSwHD3IPj2SbjVqEPcW49l72no526wuLeNDlUM+nznZOFhVwn3VEREypoHHniAbt26ERERkT2B/ZzevXtTv359GjduTK9evYiIiCjSvnz44Ye88sorREdHM2PGDFq2bElAQECRXrOkGJZlFdvralu2bCE6OprNmzcTFRVVXJctM1xnU4h78zEcsX8QMu4F3GvUBSDdaTFsuZNFRyw+ucrG4LplIxefOnWKoKBLG8KW8kv3heSlLN4X+p128VJSUvD29sYwDGJiYujatSu7d++mUqVKJd21C3ahP/ey8Zu6gjA9fQgZOwVbcFXiZj5C5p+HAPCwGXzRw8bgOgbDfnDy4R6NZImISNmzdu1aIiIiaNGiBTfccAPvvPNOmQpXF0NzsEoZ09uX0DueI3bGRGJnPkzo+Jdwq1wTN9Pgo642vO1Obl3pJM1pMbaJraS7KyIicsF69uxJz549S7obxSLPEayzZ89y/fXX07BhQyIiIrjmmms4ePBgnifYu3cv7du3p2HDhrRu3ZqYmJii7G+FYPr4EXLXVExvP+JmPIwj7hgANtPg7U42JlxhcscaF6/8mv8bHCIiIlJy8n1EOGbMGHbv3s22bdvo168fY8aMybPd2LFjGTNmDHv27GHixImMGjWqyDpbkdh8Awm963kMdw9iZzyM49QJAEzD4NV2Jo9EmNy/wcUzW5wU4zQ6ERERuQB5PiL09PTMsTpr27ZtefXVV3O1O3nyJFu2bGHp0qUADBo0iPHjx3Pw4EHCw8Pzvej48eMJCAhg4MCBDBo06DK/hfLNPvwR0j+YzIk3HsLj1icw/bNe9X2wHhiZ7jy52YO4pDSebJZBaVta5PTp0yXdBSmFdF9IXsrifZGYmFjSXZASlJiYyKlTf6/Q/78vaVzQHKzXX3+d/v375yo/cuQI1atXx27POo1hGISFhXH48OECA9b06dP1xsWFCgrCcfdLxL7xEI6Pnyd0/IvZ66k82x5C/Zzct8EDl92L19qbmKUsZZW1t4KkeOi+kLyUtfuivC4vIBcmICCgwHv2vG8RPvfcc+zdu5dnn302z/r/XZFVj6sKnz2oCqHjXsDKSCf2zUdwJidk193b3MasjjZmxLgYvcqJ06XPX0REpKQVGLCmTZvGl19+yeLFi/H29s5VX6tWLY4ePYrD4QCywtWRI0cICwsrmt5WYPaQaoSMex5XahJxMx/BmXImu25ME5MPu9r4YK/FiB+dZCpkiYiIlKh8HxG+/PLLfPrppyxbtozAwMA821SuXJnIyEjmzJnDyJEjmTdvHuHh4QU+HpRL51a5JqF3PU/s9InEvfkooXc9j+mdtfXBiAYm3na44QcnqQ4n/+1mw9Neuh4XioiURzt37izpLkgxutCfd54ruR89epRatWpRt25d/Pz8APDw8OCnn34CoE+fPkyePJlWrVqxe/duRo4cSXx8PP7+/nzwwQdcccUVeV5Mq94Wjoxj+4mb/i/soTUIufNZTE+f7LrFR1wM/N5Jp6oGX/e04V2CIassrswsRU/3heSlLN4Xhw8fpkmTJqSmppZ0V6SYeXt7s3PnzgKf2GmrnDIq48heYmc8jFv1cELGTsH08MquW3HMRf/vnESFGCzoZcPfvWRCVln8D6YUPd0Xkpeyel8cPnyYuLi4ku5GuZWYmFgqXyYICQk573QoreReRrnXakDIHVOIe/NR4v/zNMGjJ2G6ewJwVXWT7/tA7yVOrl7kZPE1NoI89bhQRKSwhYWFad5xESqrwRu0F2GZ5hHehJCxz5BxaBfx707GyszIrmtXxWRFXzv7kyyuWujgZJomvouIiBQXBawyzqNuM4Jvn0T6/h3Evz8Fy5GZXRcZYrCyn53Ys9D5Wwd/pChkiYiIFAcFrHLAs2EEIbc9ydndW4n/cCqW05Fd17SSwap+dtKc0OlbBwfOKGSJiIgUNQWscqa/irUAACAASURBVMKzSSuCb3ucs7/9zKk5L2E5/94Iun5AVsiyGdBtoYPkTIUsERGRoqSAVY54XdGG4FsfIe2X1Zz+9GUs198hq7afwdLedk6ehcc3uUqwlyIiIuWfAlY549WiA0Ej/kXq5hWc/vx1LNffYaqOv8Ez0Sav73Dx00mFLBERkaKigFUOeUd1odJN95P601IS5s3MsT/khGYmUSEGo1drSx0REZGiooBVTvlc2YNKw+4hZe0CEr+alR2y7KbBfzrbiDkNL/2iUSwREZGioIVGyzGfttdgOTJJmDsD7G4E9L8NwzCICDZ4sIXJ5K0uBtcxaRioRUhFREQKk0awyjnfjv0JuH4syT98wZnFH2WXPxVlUtMHxqxx4iq+3ZJEREQqBAWsCsCv6wD8+91G0tJPOPP9ZwB42Q3e7mhj5XGLd3crYImIiBQmBawKwr/HUPyuvoEziz4g4+g+ALrVMPm/hgYP/eTkeKpCloiISGFRwKpA/K8Zgb1KLRK+fDN70vu0NjY8bDBhnfM8R4uIiMiFUsCqQAybncABd5Cx/zfStq4EIMjT4PV2NuYesPjmoN4qFBERKQwKWBWMZ6MoPJu3J3H+f3ClnwVgaF2DvrUMxq1zciZDjwpFREQulwJWBRR43WicyYkkLf8vAIZhMLOjjcQMeGSjRrFEREQulwJWBWQPqYbfVYNI+mEujvg/AQjzNXiulcnMGBdr/1TIEhERuRwKWBWUX48bsPkEkPDN29lldzU1aVM5axuddKceFYqIiFwqBawKyvTwJODaUZzdvo6zu7cAYDMN/tPJxt5EmLpNo1giIiKXSgGrAvOK6op73StI+OotLKcDgGZBBg9HmDy3zUXMaY1iiYiIXAoFrArMMAwCB96J48QRktcsyC5/LMKkrh+MXq1tdERERC6FAlYF516zPj5tr+HMkjk4kxMA8LQbvN3JxroTFm/t1KNCERGRi6WAJfj3vRWAMws/yC7rXM1kTGOTh392cTRZo1giIiIXQwFLsPkG4n/NCFI2LMnepxDghdYmvm4wbp0ze2sdEREROT8FLAHAt2M/7FXCSJg3MztMBXoYTG9vY/4hi3kHFLBEREQulAKWAH/tUzjwDjIOxJC25cfs8oF1TK6vbXD3Oien0xWyRERELoQClmTzbBiJZ4v2JM5/N3ufQoDpHWykOmDiT84S7J2IiEjZoYAlOQReNwZnSiJJyz7LLqvhY/BCa5P/7LZYeVxvFYqIiJyPApbkYA+uil+3wSStmIcj7nh2+ZgmJh2rGIxe5eSsQ48KRURECqKAJbn4dR+GzTeAhG/eyS4zjay1sQ4lwzNbNYolIiJSEAUsySVrn8LbOfvr3/sUAjSpZPBYpMmLv7jYHq9RLBERkfwoYEmevCK74F63GQlf/r1PIcDDLU0aBmRto+N0KWSJiIjkRQFL8pS9T+HJozn2KXS3Gfyns42NsRbTY/SoUEREJC8KWJIv95r18Gl3DWcWf5S9TyFAuyomdzU1eWyji0NJGsUSERH5XwpYUiD/PreCYeTYpxDguStNKnnAnWu1jY6IiMj/UsCSAtl8AwjofXPWPoVH9maX+7sbzOxgY/ERi89+V8ASERH5JwUsOS+fDn/tU/jlmzlGq/rXNhlSx+Ce9U7izypkiYiInKOAJedl2GwEDrwz1z6FAK+3t5Hpggc2aBsdERGRcxSw5IJ4NozAq2VHEub/B1d6WnZ5VW+DaW1sfLDXYtkfeqtQREQEFLDkIgRcezuu1CSSlv03R/ltjQy6VjMYu9pJqrbRERERUcCSC2cProrfVYNJ+mEejrhj2eXGX9vo/JEKT2/WKJaIiIgCllwUvx5DsfkFkvD1OznKGwQYPBVl8vKvLrbEaRRLREQqNgUsuSim+1/7FO5Yz9ldm3PUPdjC5IpKcPsqBw5toyMiIhWYApZcNK/IzrjXa07CV7Ny7FPoZmZto/PLKXh1hx4ViohIxaWAJRctxz6Fq7/NUXdlqMmEK0ye3OTiYLJRQj0UEREpWQpYcknca9TFp30fziz5CGdSQo66Z1qZVPaCR37xLKHeiYiIlCwFLLlk/n1uAdPkzMLZOcp93Qyeb21j6Z92NpzQo0IREal4FLDkktl8/AnofQspP32XY59CgCF1DBr7O3lKyzaIiEgFpIAll8WnfV/sVWvn2qfQZhr8q0kGS/+wWPOnQpaIiFQsClhyWXLsU7h5RY66fjUctAhCo1giIlLhKGDJZfNs0DJrn8Jv382xT6FpwKRoGz8cs/jxmEKWiIhUHApYUigCrhudtU/h95/lKL+utkFkcNYo1j8fIYqIiJRnClhSKOxBVfDrNoSkFV/iiM25T+HkaBur/rT44ZgCloiIVAwKWFJo/LoPydqn8Ju3c5T3DTO4MtTgSY1iiYhIBZFvwJowYQLh4eEYhsGOHTvyPUF4eDiNGzcmIiKCiIgI/vvf/xZJR6X0M909CbhuNGd3bODszk3Z5VmjWCbrTlgsPaqAJSIi5V++AWvw4MGsWbOG2rVrn/ckc+fOZdu2bWzbto1hw4YVagelbPGK6JTnPoW9ahq0q6xRLBERqRjyDVidO3emZs2axdkXKQcMwyBw0J04Yv/AsXFpjvLJrUx+jrVYdEQBS0REyjd7YZxk+PDhuFwu2rRpw9SpUwkNDS2w/fjx4wkICGDgwIEMGjSoMLogpYlnIPbIrmSs/pr46B4Ybu4ARHpCuxAvHvvJSVufVAztBV0hnT59uqS7IKWQ7gvJS1m6L4KCgnJ8fdkBa9WqVYSFhZGZmcnjjz/OrbfeyqJFiwo8Zvr06URFRV3upaUUc/S5mT+3rsBj70Z8O/bPLn+ujYurFjpZkxTIdeF6x6Ki+t//EImA7gvJW1m9Ly77N1xYWBgAbm5u3HvvvaxevfqyOyVlnz2kGrambUn6YS6W05ld3rW6yVXVDJ7a7MSluVgiIlJOXVbASklJISEhIfvrTz/9lMjIyMvulJQPbu374zx1grRtq3KUT4o2+eUUfHVQAUtERMqnfAPWuHHjqFmzJkePHqVHjx7Ur18/u65Pnz5s2rSJEydOcNVVV9GiRQuaN2/OypUr+fDDD4ul41L6mdXC8WgcTdLyL3K8OdipmsnVNTSKJSIi5ZdhFeM781u2bCE6OprNmzdrDlYFcOrUKbzjjxA341+EjJ2CZ5NW2XXrT7hoP9/JZ91sDKunuVgVyalTp8rsnAopOrovJC9l+b7QbzYpUh71W+AW1pCk5Z/nKG9XxaR3LYOntzhxujSKJSIi5YsClhQpwzDw7z6U9H3bST+4K0fdpGiTXQnw2X4FLBERKV8UsKTIeTZvhz20Bkk/5BzFujLUpH+YwaTNThwaxRIRkXJEAUuKnGHa8Os+hLO/rifzxJEcdZOibew9Ax/vU8ASEZHyQwFLioV3q26Y/kEk/fBFjvLIEIMB4QaTtzjJ1CiWiIiUEwpYUiwMuzt+Xa4nddMPOBPictQ9HWVjfxJ8uEcBS0REygcFLCk2Pu37YLh5kLTyqxzlLYINhtQxeGarkwynQpaIiJR9ClhSbExPH3w79iNl3SJcqUk56p6KsnE4Gd7f4yqh3omIiBQeBSwpVr5drsdyOkheuzBH+RVBBjfUM5iy1UW6RrFERKSMU8CSYmXzq4RP654kr/wKKyM9R92TUTaOpcJ/dmkUS0REyjYFLCl2ft0G4UpJIuXn73OUNw40GF7P4LltLtIcGsUSEZGySwFLip09pDpeEZ1IWjEPy+nMUfdElI0TafC2RrFERKQMU8CSEuHXfQjO+OOk/bI6R3mDAINbGhhM3eYiVaNYIiJSRilgSYlwr1kfj0ZRJC3/AsvKGaQej7QRfxbejNEoloiIlE0KWFJi/LoPJfOP30nftTlHeV1/g/9rZPDCLy6SMzWKJSIiZY8ClpQYjwYtcavVINf2OQCPRdhIyIAZv2kUS0REyh4FLCkxhmHg12MY6Xt/IePQ7hx1tf0Mbm9k8uJ2F2cyNIolIiJliwKWlCiv5u2wh9YgafnnueoejTBJzoQ3NIolIiJljAKWlCjDtOHbbTBpv64j88SRHHU1fQ3GNjGZtt1FokaxRESkDFHAkhLnc2V3TL9KJK+Yl6vu4ZYmZ53w6q8axRIRkbJDAUtKnGF3x6/LAFI2LseZGJ+jrrqPwZ1NTF7+1cXpdI1iiYhI2aCAJaWCT4c+GG7uJK38Klfdv1qaZLrgZY1iiYhIGaGAJaWC6emDb4e+pKxdhCs1OUddFW+D8VeYvLrDRfxZjWKJiEjpp4AlpYZvl+uxnJkkr12Qq+6hFiaWBdO2axRLRERKPwUsKTVs/kH4tL6a5FXfYGWk56gL9TKY0Mzkjd9cnEzTKJaIiJRuClhSqvhdNRhXciIpG5flqnuguYlpwEsaxRIRkVJOAUtKFXtodbxadiTph7lYTmeOumBPg3ubmcz4zcWfqRrFEhGR0ksBS0odv+5DcMYfJ+2XNbnq7mtu4m6DF37RKJaIiJReClhS6rjXaoBHw0iSfvgcy8o5UlXJw+D+5iZv7nRxLEWjWCIiUjopYEmp5NdjKJlHfyd9z9Zcdfc0M/G2w9RtGsUSEZHSSQFLSiWPBhG41WpA0rLcm0AHuBs82Nzk7V0ujiRrFEtEREofBSwplQzDwK/7UNL3biPj8J5c9XdfYeLnBs9pFEtEREohBSwptbxatMceWoOk5blHsfzcDSa2NHl3t4uDSRrFEhGR0kUBS0otw7The9Ug0ravJfPk0Vz145qaBLrDs1udeRwtIiJSchSwpFTzubIHpl8gySvm5a5zM3i4pcn7eyx+P6NRLBERKT0UsKRUM9zc8e08gJSfl+FMjM9Vf0dTk1BPePhnjWKJiEjpoYAlpZ5vh74Ybm4krfw6V5233eDltjbmHrBYeFgT3kVEpHRQwJJSz/TywbdDP1LWLcSVlpKr/oZ6Bj1rGNy11klKph4ViohIyVPAkjLBt/N1WJmZJK9dkKvOMAxmdrRxMg2e3qJRLBERKXkKWFIm2AKC8Wndg+SVX2NlZuSqr+dv8GSUySu/utgWr1EsEREpWQpYUmb4dhuMKzmBlI3L8qx/oLlJ40AYu9qJ06WQJSIiJUcBS8oMt9AaeLXoQPIPc7Fcud8adLcZzOpo4+dYi7d26lGhiIiUHAUsKVP8ug/FEXeMtO1r86zvUNVkbGOTRza6+CNFo1giIlIyFLCkTHEPa4hHwwiSln2OZeUdoKa2NvG2wz3rtTaWiIiUDAUsKXP8ug8l8+g+0vdszbO+kofBq+1szDtg8e0hPSoUEZHip4AlZY5Hw0jcatYnafkX+bYZVtegV02D8eucJGttLBERKWYKWFLmGIaBX/ehpO/ZSsaRvfm2mdnBRmwaPL1Zo1giIlK8FLCkTPJq2QFbSDWSln+eb5u6/gZPRZm8usPF1jiNYomISPFRwJIyyTBt+F01mLRf1pAZ+0e+7e5vYdIkEMZobSwRESlGClhSZvm0vhrTN5DkH+bm28bNNHi7k41NcRYzY/SoUEREiocClpRZhps7vl2uJ+XnZTgT4/Nt166KyR1NTB7b5OJoskaxRESk6ClgSZnm26EfhptbgW8UAky90sRHa2OJiEgxUcCSMs308sGvxzCS13xL5p+H8m0X6GHwWjsbXx60mK+1sUREpIgpYEmZ59d1APagKiR8+Va+q7sDDKlr0LuWwfi1WhtLRESKlgKWlHmG3Z2AAWNJ37OVs/nsUQhZa2PNaG8j7iw8uUmjWCIiUnQUsKRc8LqiDZ5NW5Pw9du4Ms7m266Ov8HT0Sav/eZii9bGEhGRIpJvwJowYQLh4eEYhsGOHTvyPcHevXtp3749DRs2pHXr1sTExBRJR0XOJ3DAHTjPnC5w2QaA+5qbXFFJa2OJiEjRyTdgDR48mDVr1lC7du0CTzB27FjGjBnDnj17mDhxIqNGjSr0TopcCHtodfy6DuDM8s9xxP+Zbzs30+Dtjja2xFnM0NpYIiJSBAyroFnBQHh4OAsWLKBZs2a56k6ePEnDhg2Ji4vDbrdjWRbVqlVjw4YNhIeH52q/ZcsWoqOjadeuHQEBAQwcOJBBgwYV2jcjpcvp06epVKlSsV7TyjjL2RkPYtaoh8fQ+wps+9BWD/572I31V6dQw1sjWcWlJO4LKf10X0heytJ9ERQUlONr++Wc7MiRI1SvXh27Pes0hmEQFhbG4cOH8wxY50yfPp2oqKjLubSUEf97wxWH1OtHc+qjF/COPYhno/zvs5c7WSz63MGTMX581fOy/lWQi1QS94WUfrovJC9l9b647EnuhmHk+Po8A2IiRc4rqivudZtlLdvgdOTbLsDd4PX2Nr4+ZPH1QT0qFBGRwnNZAatWrVocPXoUhyPrl5hlWRw5coSwsLBC6ZzIpTAMg8BBd+E4eZTk1fMLbDu4jkGfWgZ3r3OSlKH/ORARkcJxWQGrcuXKREZGMmfOHADmzZtHeHh4gY8HRYqDe426+HTow5klc3Amnc63nWEYzOhg41Q6PLlZo1giIlI48g1Y48aNo2bNmhw9epQePXpQv3797Lo+ffqwadMmAGbNmsWsWbNo2LAhzz//PO+++27R91rkAgT0vhXDtJH47XsFtgv3M5gUbfL6by42x2oUS0RELt953yIsTOfeIty8ebMmuVcAp06dKvHJiclrF5LwxRuE3vsqHuGN822X6bK48isHNhN+us6O3TTybSuXpzTcF1L66L6QvJTl+0IruUu55tPuGtxq1iPhy5lYrvwfAbqZBrM62dgaB9N/06NCERG5PApYUq4Zpo3AgXeReXgPqT8vLbBtm8omdzU1eXyTiyPJelQoIiKXTgFLyj2Pulfg3aobiQvex5WaXGDbZ6808XeHu9c5i6l3IiJSHilgSYUQ0H8UVmYGZ5bMKbidu8Eb7W18o7WxRETkMihgSYVgCwjGv+dNJK+ZT+bxgwW2HRhu0C/MYLzWxhIRkUukgCUVhm+X67EHVyPhyzcL3HHAMAymt7dxOh0e36RRLBERuXgKWFJhGHY3AgfeSfreX0j7ZU2BbWv7GUyONnnjNxcbYxWyRETk4ihgSYXi2aQVnle0IfGbd3BlnC2w7T3NTFoEwdjVThwuPSoUEZELp4AlFU7ggLE4k06TtOzzAtvZTYO3O9nYFg9vaG0sERG5CApYUuHYQ6rjd9Ugkn74Akfc8QLbtq5sMv4Kkyc2uTistbFEROQCKWBJheTX4wZsvgEkfPPOedtOaWUS4A7j1zoLnBwvIiJyjgKWVEimhycB147m7K/rOLtrc4Ft/f9aG+vbwxZfHVTAEhGR81PAkgrLK7IzHvVbZC3b4MgssO2AcIP+YQZ3r3OSqLWxRETkPBSwpMIyDIPAgXfiiD1G8qpvztt2egcbyZnQfaGTE6kKWSIikj8FLKnQ3KrXwadjP8589zHOxPgC24b5Gqzsb+ePFIsO3zr4/YxCloiI5E0BSyq8gN43Y9jdSVzw/nnbRgQbrL/Ojs2A9vMdbI5VyBIRkdwUsKTCM7398O83ktSNy0g/EHPe9uF+BmuvtVPHz6DLAgdLj2qNLBERyUkBSwTwadMTt1oNSJg3E8vlPG/7EE+D5X1sdKlm0HeJkzl7FbJERORvClgigGHaCBx4J5lH95Hy09ILOsbHzeDrnjZubmBw849Opm3XOlkiIpJFAUvkLx51muJ9ZQ/OLJiNKzXpgo5xMw3e7WzjsQiTh35ycf8GFy6FLBGRCk8BS+QfAvrfhuXIJHHxRxd8jGEYTLnSxowOJq/tcHHTD07SnQpZIiIVmQKWyD/Y/IPwv2Y4KWsWkHFs/0Ude1dTG3N72Pj6kEWfJU7OaEFSEZEKSwFL5H/4droWe2h1Eua9edFzqgbWMfm+t40tcRadv3VwXAuSiohUSApYIv/DsLsROPBOMn7/lbStqy76+E7VTFb3txOXDu2/cbA7QSFLRKSiUcASyYNn42g8m7cncf47uNLPXvTxzYIM1l9rx9sOHeY7+OmklnEQEalIFLBE8hF43WicyYkkLfvsko6v5Wuwur+dJoEGVy1wsvCwQpaISEWhgCWSD3tINfy6DSHph3k44o5d0jmCPA2W9rHRq6bBdUudvLdbIUtEpCJQwBIpgF+Podj8K5Hw1axLPoeX3WBuDxu3NzIZtcrJs1u1IKmISHmngCVSANPdk4DrRnP2t59Ii9l4yeexmQZvdjSZHG3y+CYX49e5cLoUskREyisFLJHz8GrZEY8GLUn86i0sR8Yln8cwDJ6IsvFOJxtv7XQxdLmTsw6FLBGR8kgBS+Q8DMMgcOCdOOKPk7zym8s+3+2NTb6+2sbiIxa9Fjs5na6QJSJS3ihgiVwAt2rh+Ha8ljNLP8Fx6sRln69/bZPlfW3sOG3R6VsHR5MVskREyhMFLJEL5H/NCExvP2LfeIjME0cu+3ztqpisvdZOUia0m+8g5rRClohIeaGAJXKBTG9fQu/5N4aHJ7GvP0jG4T2Xfc7GgVkLkgZ5QMdvHaz9U8s4iIiUBwpYIhfBHhhK5bv/jT2kGrEz/sXZ3Vsu+5zVfQxW9bfTMsigxyInXx9UyBIRKesUsEQukunjR8hdz+Ne5wri3n6S1G0Xv1/h/wpwN1jS20b/MINBy5zM2ukshJ6KiEhJUcASuQSmhycho5/GO6ITpz6YSvKaBZd9Tg+bwWfdbYxranLHGhdPbHKS7tS8LBGRsshe0h0QKasMm51Kwx/C9AkgYe50XMmJ+PW6CcMwLvmcpmHwWjuTGt7w8EYXs3a6GNXIZGwTk3C/Sz+viIgUL41giVwGwzQJGDAW/74jObPkIxK+fBPLdXlzqAzD4F8RNmIG27mxnsmbO13U/cxB3yUOFhzSCvAiImWBApbIZTIMA/+rbyBw6ARS1izg1EcvYDkyL/u8TSoZvNbexh832Xmnk40TadB/qZN6/3Xw3FYnJ1IVtERESisFLJFC4tu+D0EjHyVt+zri3nkKV3paoZzXx81gVGOTTQPs/Hy9jW7VDZ7Z6qLWpw5uWO5g5XGXNo8WESllFLBECpF3y46E3DGFjIO7iJ3xMM7kxEI9/5WhJu91sXNsuJ0XW5tsi7fousBJs7kO3tjhJDFDQUtEpDRQwBIpZJ4NWhI6/gWcp/4k9vUHcZw+WejXqORhcG9zGzuH2Fnex0bTSgb3b3BR/WMHo1c52BKnoCUiUpIUsESKgHutBoTe8zKWI4PY1+4n88/DRXIdwzDoVsPkix52Dt9k5+GWJkuOWkR/5aDN1w5m73GR5lDYEhEpbgpYIkXELbQGle95GcPLl9jXHyD94K4ivV41b4MnomwcuMHO11fbqOQB/7fSSY1PHNy/3smeBAUtEZHiooAlUoRsAcFUvvsl7FVqETfzYc7u2lzk17SbBteFmyzpbWffMDu3NzL5cK+LRl846LHQwbwDLjK11IOISJFSwBIpYqa3HyF3PodH/RbEvfMUqVt+LLZr1/M3eLGNjaM32fmoq400Jwxe5iT8UwdPbXZyNFlBS0SkKChgiRQD092T4FFP4h3VlVMfvUDy6vnFen1Pu8GIBiZrr7Xzy0A719Y2eflXF+GfORiw1MH8Qy5i0xS2REQKi7bKESkmhs1OpRvvx/TxJ2HeTJzJifhfM+Kytta5FC2CDd7saOPF1iZz9rl4c6eL65ZmbS5d0weiQwyiQgyigg2iQw2qeWuLHhGRi6WAJVKMDNMk4LrR2Pwqkfjtu7iSEwkcdCeGaSv2vvi5G9zZ1MYdTUz2J8GWOIvNcRZb4ixe2+HiVHpWu6pef4euc3/X9KHYg6GISFmigCVSzAzDwK/7EEwff07/9zVcKYkEjXgIw+5eYv2p5581X2tI3awyy7I4nEx24NoSbzFrl4uTfy1OH+pJ1ijXudAVbBDup9AlInKOApZICfFp2wvT24/4D6cS9/ZTBN/2BKand0l3C8gKSrX9oLafwcA6WWWWZXEs9e/QtTnO4oM9LqZuy6qv5AFRwTlHuur5g6nQJSIVkAKWSAnyatGe0DueJe4/TxM741+EjH0Gm29gSXcrT4ZhUMMHavgYXFv77/I/U/8a5fordP13v4uXtmfV+btB5D/mc0UFG4RoLr2IVAAKWCIlzKN+C0LHv0TcrMeJfe0BQu58DntQlZLu1gWr6m3QJ8ygT9jfZbFpFlvj/57T9c0hF6/syKqzG77U8Mmklo9BLV+o5WMQ5gu1fI3ssmAPPW4UkbJNAUukFHCvWY/K9/yb2DcfywpZd0zBrVp4SXfrkoV6GfSsadCz5t9lp9P/Guk6lsIpy5sjKRZHkmHDSRdHUyDT9XdbLxvZ4evvvw3CfM4FsaxJ+iIipVW+AWvv3r3ceuutxMXFERgYyOzZs2natGmuduHh4Xh6euLp6QnAI488wrBhw4quxyLllD2kOpUnTCNu1hOcfONBQkZPxqNO7n/nyqpKHgbdaxhEemUSFJTzrUmXZXEiDY4kWxxJ+fvvw8kWOxNg6VEXx1Phn08XA9yhlg+E/WPk69zfYb5Zbzp62BTCRKRk5Buwxo4dy5gxYxg5ciRz585l1KhRrF+/Ps+2c+fOpVmzZkXWSZGKwhYQTOj4F7PmZM18GN+21+DbZQD2kGol3bUiZRoG1byz9lNsnU+bTJfFsRQ4kpL1hmN2GEux+DnWxdwDEJ+e85jKXllre9X0yQph2f/sm/V3De+sRVhFRApbngHr5MmTbNmyhaVLlwIwaNAgxo8fz8GDBwkPDy/O/olUOKa3L6F3PMuZ5Z+Tsub/27vz4Ciuw0/g39fdc1/SSAJJIFnIIMAYG9kYDOEUxibYcTAyv3iDXYYilaRiJwXZxFXrUIXLtSaVSuL8Etv128Sp5UfMFms7wHoTnMPGGOEz2OCDtYPwIUuKLAQanTMjaab77R89aml0cY0YSfP9VHV19+s3w5txu/ny+s3rP6PzDYAoTAAAGd1JREFUtT/Ddf1X4F1RCUfJrHQ3L21sSt8vG4cTiUvUd5qhq7cnrD4M1IclqhrNW5EtA0JYnrNfCPP2C2GJ9RQP4GIII6KLNGTAqqurQ2FhITTNPCyEQHFxMWpra4cMWBs3boRhGFi4cCF++tOfIi8vb8Q/9MEHH0QgEMD69etRWVl5+Z+CxqSWlpZ0N2F8W3g7HDfcgvj7R9H11ouI/vtWKMWzoC26HWpZOYQYn0+6Gu3zIhdArgsodyV2BgjHgYaoQENUQUNU4F8RBV8m9o/8y1y39CQHqqDdwBSXRKFLosBlYIpbotBloNAlMcVloMAl4eaI1svC6wUNZTydF8FgMGl/2EvCwF/wSDn0b6urqqpQXFyMWCyG7du34/7778eLL744YiOefPJJ3HDDDRfaZhrHBp5wdAlu/TfIWyrRdfJtdBz+I3qe/SW0vCnwrqyEZ/4qCLsj3S28aOk8L4IAis5Tp7cnrD5s9oDVhRWrJ+xEm8SfGgbfjgzYzVnv891ixHWeE1AV9ogNhdcLGsp4PS+GDFhFRUWor69HPB6HpmmQUqKurg7FxcWD6vaW2Ww2bN26FWVlZaPbYqIMJBQVrusWw3XdYnR//hE6X92P1uefQPuLu+Fdcic8S+6A6g2ku5kThlsTKMsCyrKGD0LReN/tx7pOoDEq0RjpW38QMtAYHXxLUhFmyOofugrcAvluIN+VvPbbOF0F0Xg1ZMCaNGkSysvLsWfPHmzatAn79u1DSUnJoNuD4XAYsVgMWVnmxIh79+5FeXn5qDeaKJM5pl0Dx7RrED/XgI5XD6Dj0HPoOPQc3AtWw7viLtjypqS7iRnBpQnMCAAzAiMHoG7d/IVkY0SiMYqkENYYlahuA6oaDTRGgKie/FqnOrhXLM8J2BVAVQBNAKowt9XEtqYIa9vcR9L+oLLEtmZt973epgAONbEogFMz1+yBIzq/YW8R/va3v8WmTZuwc+dO+P1+7N692zq2du1aPProowgGg6isrISu65BSorS0FH/4wx+uSMOJMp2WW4jsux+A/6v3Ifz6n9FZ9X8RfuMgnHMXwbfy7gk1xcN45lDNiVSLvSOHEiklOmKDA1j/cPaPs+bzIGMGoEsgnlhbiwHEr8BM+aowQ5ezX/jq2xfJx3rDWb9jA1/r1IBY1IbAOQNaIthpiQDZu7bKrH1x3jqakigTZs9hKnsDe4fNSAC9I2gk+qYS6S1TEmGVPZGZR8jhBleNguPHj+PGG2/Eu+++yzFYGSAUCo3be+fjkYz1IPLOIXQc3o94Ux3sJbPhW1kJ59xFEIp6/je4QnhejD5DSitw6dIMXXq/MDYomA0qk9ANM8h1G0C3bi5deu+27Ns2Rj7WFe9fRw5+r8Sx/hPNjpbeECaEGYCsMIQBIWmYwHQ5f1mqYnBoHBgGtX49jOb+4BDZ/zUD39MMcwJKIlAqQN/2EGXqMOXDlfXWFxCQkDAS34khze/HSHxPSeUY+diQdfuVRaJdsDmcg87N+IB/WAw8n4c79633GFCuCuCf/2a7jP/Cg/F3L0QThLDZ4Vn0VbgX3oauj4+h45U/onnXf4eaWwDfivVwL1gNxe5MdzPpClASf8naLvmHple+t6W5OYRAdjZihhn24jJ5PVR5zJDJZVb5cO8hk4KcgBm2htzurZMIFH3bF/5awAwWZngduq1Drq1tOeyxiN4XHno/swSgGxJGIvwkLcD5ywbs6yMcUxLfhdkzaIYxIQaUY+RjQ9YdUCYNFXbN6Hc7Wwx5a7z/LW+bAqjacLfIBdQBt9FVAdhH4d+gDFhEE4xQFLjmLIRrzkL01J5Cx+H9aN33H2j/yzPwfOUOeJd+DaovO93NJEoiEuPHtIsKhbztNtGFQh3jtsebAYtoArMXz0TO/f8N8Ts2o/PI/0Hnq/vR8crz8Nx0C7wr1sM2+XwTFhAR0aVgwCLKAFpOPrLWfxf+NRvR+caL6Kx6AeE3/wLnnIXwVdwNe+m1HIRLRJRCDFhEGURx++C/5RvwrbgLkXdfRcfhfTj7xI9hm3o13DdWwDVvGbTskZ/EQERE58eARZSBhGaHZ+GtcC9Yja6P30H4rb+g7eB/ou2Fp2Gfdg3c5cvhmrcUqn98jn0gIko3BiyiDCaEgOuam+C65iYYXWFET76F6PEjaH3habQe+B9wTL8OrnnL4Lp+CWeKJyK6CAxYRAQAUJweeOavgmf+KhiRDkQ/eAORE0fQuu8ptO57Co4Z8+C+YQVccxdBcfvS3VwiojGNAYuIBlHcPnhuvg2em2+D3tmK6PuvI3riCFr+96/Q8txv4Jx1I1zly+C6dhEUpzvdzSUiGnMYsIhoRKo3C96v3A7vV26H3taMyHtHEX2vCi17fo4Wmx2u2TfBVb4czjkLOJEpEVECAxYRXTA1kAPf8nXwLV+HeEsToieqEDlxBKHdOyHsTjjnLIS7fDmcs+dD2Ozpbi4RUdowYBHRJdGyJ8FXcTd8FXcjfrYBkfeqED1xBM3/81EIpxuuuYvhKl8GZ1k5hJbaZ3wREY11DFhEdNm0vEL4V98D/+p7EGusReTEEbN369jLEG4v3Nctgat8GRzTr093U4mIrggGLCJKKVt+MQJfvQ/+Nfci1vA5oieOIHKiCuG3/grFmwUxoxyR6xbBMeN6Tv1ARBMWAxYRjQohBOxTSmGfUgr/7ZsQq6s2g9aHbyJ04jAAwDb1ajjKyuEsK4e9dA4HyRPRhMGARUSjTggBe/FM2ItnwlhaCb+io7v6PXRXn0DknUPofOWPgGqDo/QaM3DNLIdt6nQIRU1304mILgkDFhFdcVpWHrQFq+FZsBpSSsQbv0BXInB1vPws2g/+J4TLC+eM6+EoK4djZjm03EI+kJqIxg0GLCJKKyEEbAUlsBWUwLd8HaQeR88Xp9BdfQJd1SfQuv8/AEOHmj0JjrJ5cJaVw1E2D6ovO91NJyIaFgMWEY0pQtXgKJ0DR+kc+NfcC6Mrgu5PP0R39Xvoqj6ByNt/BwDYCqeZvVtl5XBcfS0UhyvNLSci6sOARURjmuJ0wzVnIVxzFgIA9PaQFbai71Wh89X9gKrBXjIbzrJ5cJSVw148E0Ll+C0iSh8GLCIaV1R/EO75FXDPrzDHbzXVJ24nvoeOw/vR/pdnIJxuOKZfB0fpHNiKymAvmg7F6Ul304kogzBgEdG4JYSAbXIRbJOL4F16J6Suo6fuNLqrT6C7+gTa//q/IHu6ACGg5U2FvbgMtqIZsBeXwT7lagi7I90fgYgmKAYsIpowhKrCUTILjpJZwK3/BdLQET9Th57aavTUnUZPrTkXF/QYoCiw5ZfAVjwD9qIyM3wVlPCxPkSUEgxYRDRhCUW1fqHoWXgrAEDGY4h9WYOeutOIJYJX5B8vAYYBqDbYpkwze7gSoUubXMT5uIjoojFgEVFGEZoN9qIZsBfNABavBQDInm70NHxmBa7u0x8g/PpBQEoIuwO2qdP7ermKZphzcilKmj8JEY1lDFhElPGE3QFHyWw4SmZbZUZXBLH6T6xbi9H/9xY6jxww6zs9ZkjrN6ZLzZ7EiVCJyMKARUQ0BKX3l4jTr7PKjHAHeupPJ8Z0VSPy7ivQDz0HABAON2z5xdAmF8OWX5RYX2UGL/Z2EWUcBiwiogukeHxwzrwBzpk3WGV6ewg9dacRb6xFrPELxBq/QPT9o5DdUQCAsDmgTZ4K2+RiaPlXJdbF0HIKOFcX0QTGgEVEdBlUf9CcBDUxESoASCmht55DvPELxM7UmuHrTC2iHx2DjHYmXmiDNmkKbJOLzZ6v/GIzfOUVQmj2NH0aIkoVBiwiohQTQkDLzoOWnQfn7PlWuZQSRkcLYo21iJ+ptcJX52sfwOhsNSspCrTcQitwWT1ek6ZCsTvT9ImI6GIxYBERXSFCCKj+IFR/ECibl3RM72xD/EwdYo1fmOsztYj842Xobed6Xww1OBm2yUXQ8qZCyy0wg1heIdTsybzdSDTGMGAREY0BqjcA1RuA4+prk8qNaNjs6bLCVy26Pj6GeHMjoMfNSooKLTgZaiJwabmFfQEsZzJvORKlAQMWEdEYprg8g6aQAABp6NBbziJ+rgHxc1+a67MN6D79PsJv/RWI9ZgVhQI1Oy8RugqTe75y8nnbkWiUMGAREY1DQlGh5eRDy8kHZiYfk4YBvb0Z8bMN0PsFsJ6ajxF555D5fMYENZALNbegX89XXxBTnO4r/KmIJg4GLCKiCUYoCrSsPGhZecCM65OO9Q607+3x6u0Bi/3rM0TfOwrZFbHqKt4sqMFJ0LInQU0sWnaeta14/JxclWgYDFhERBmk/0B7R2nyeC8pJYxweyJ0NUA/9yXiLU3QW84i1vA24i1NQDzW9152B9SsSf1CWF4ihCUCWVYuhMq/Zigz8cwnIiIAifDVO9h+wJgvIBHAOtugt5xBvOUs9NAZcxxYSxN66k5D/+ANGOG2fm+oQA0EE2ErzxyIPyCEKS7PFfyERFcOAxYREV0QIQRUXxZUXxbsxTOHrGN0d0FvNcNXvPUs9JYm6KEmM4R98U/orecAQ+97T6cHWnYedE8WWvIKoPiDULNyzV62QC7UQI55K5KPG6JxhgGLiIhSRnE4oUwugm1y0ZDHpaFDbw9BbzHDVzzUBL21CV1NDeip/wR62zkYHa2AlH0vUjXrtmZv+FICuWbvWCKEqYEcDsqnMYUBi4iIrhihqH0D8KddY5WHQiEEg0EAgNR16B0hGG0h6G3N0NvOmaGs1VzHztRCbwv1PXao970dLits9YWwnOTFH4TQbFf0M1NmYsAiIqIxRahmCENW3oj1jO4uGO0h6O3NVvjqXcdbmqDX/NOcCb/fwHwAUDx+KL4sqL5gYp0NxZdt3v709yvzZnGGfLpkDFhERDQuKQ4nlDxz0tThSCkhI519IaytGXpHC4yOFujtLdDbmhGr/xR6R8ugHjEIAcXth+JPhC8riPUFMsUfNNfeAITCMEZ9GLCIiGjCEkJAeHxQPD7YCkpGrCvjPdA72mB0hKB3tJq9Yx2tZhjraEG89RyM2mroHa2QXeGBfxAUT8AMW75sqH4zhCmeAFSvH4onYPacefxQvAEoLi8H7k9wDFhEREQAhGaHlp0HZI98axIAZKwn0RPWCj0RwHp7xYyOFsRDTTC+OAU93AYZ6Rz8BkKB4vYlQpfPDGBeP9SBQczTW+aDcHoYysYRBiwiIqKLJGx2aMHJQHDyeetKXYcR6YARboMRbocRbocebofR2bdvhNsQa6hBd2K7/4z6FkUxb1kmesTU/kHM7RuweM21ywthd4zCN0Dnw4BFREQ0ioSqWvOHXSgZjyWHsX5BzOg013q4HfGWJiuo9X/GZBKbHYrLOyh4Je27fRBD1OEg/0vHgEVERDTGCM1mTS1xoZNKyHgMRrQTRqTT7DGzlk7IaKe1bUQ6ED/XACPckajfAejxodvhdJthzOOD4uoLX1a5ywPh8kBxesxA5kqUOz0QDldG39JkwCIiIpoAhGaDmviV48WQUkL2dJsBLNoBGUkOY0a0MymMxUNnILsi5n40PGw4gxAQTg+URBgTLg8UK4wlgpnLC8XpNnvPEsdFIqwprvE9cSwDFhERUQYTQkA4nFAczgsa4N+flBKI9cDoCsOImouMdsJIBDA5RHk81AiZKDO6wkOPN+ul2tDldEM4XVAcLrPnzOE2e8ecLmtbcSavrW2n23ydww1hd0AIcZnf1oVjwCIiIqJLIoQA7A6odgdUf/CS3kMaOmRX1OwR64okglgYRiSMztBZuFRh9ph1R621EWmHEToDaZVFILujyY9YGtRYxQySTrcZuBwuKE4zfClON7K/+V9TGsAYsIiIiChthKJCuL1Q3N5Bx7pDIfiDFxbcem91yu6IGdS6o9a6bzsCoyux3x0xw1lXFHp3JOW9WwxYRERENO713uqEw3nJvWmplLnD+4mIiIhGCQMWERERUYoxYBERERGlGAMWERERUYoNG7BOnz6NxYsXo6ysDAsWLMBHH310WfUo8+zbty/dTaAxiOcFDYXnBQ1lPJ8Xwwas73znO/j2t7+N6upqPPTQQ9iyZctl1aPMs3///nQ3gcYgnhc0FJ4XNJTxfF4MOU1DU1MTjh8/jr///e8AgMrKSjz44IOoqalBSUnJRdfrFY1GAQCbNm2C1+tFRUUFVq1aleKPRGNFc3MzDh8+nO5m0BjD84KGwvOChjKezotAIIBZs2bB7U484kcO4Z133pGzZ89OKrvpppvkkSNHLqlerz179kgAXLhw4cKFCxcuE2559913rcwz7ESjA2c0lcNMP3+h9QDgtttuw549e1BSUgKXyzVsPSIiIqLxZtasWdb2kAGrqKgI9fX1iMfj0DQNUkrU1dWhuLj4kur1ys3NxcaNG1P4UYiIiIjGniEHuU+aNAnl5eXYs2cPAHMUf0lJyaBxVRdaj4iIiCiTCDnMPb1Tp05h06ZNaG5uht/vx+7duzFnzhwAwNq1a/Hoo49i/vz5I9YjIiIiykTDBiwiIiIiujScyZ2IiIgoxRiwKOVKSkowa9YszJs3D/PmzcOzzz6b7iZRGvzgBz9ASUkJhBA4efJk0jE+ASKzjXRu8PqRebq6urBu3TqUlZVh3rx5WLNmDWpqaqzj4/Z6MeSEVUSX4aqrrpIffvhhuptBaXbkyBFZV1c35PmwcuVKuWvXLimllM8//7y8+eab09BCSpeRzg1ePzJPNBqVBw8elIZhSCmlfOKJJ+Tq1aut4+P1esEeLCIaFcuWLcPUqVMHlfc+AeLee+8FYD4B4vPPP0/6FytNbMOdG5SZnE4n1q5da82refPNN+Ozzz4DML6vFwxYNCo2btyIuXPn4lvf+hbOnj2b7ubQGFJXV4fCwkJomjkNnxACxcXFqK2tTXPLaKzg9SOz/eY3v8HXvvY1AOP7esGARSlXVVWF999/H8ePH0dOTg7uv//+dDeJxpiLeQIEZRZePzLbzp07cfr0aTz22GNW2Xi9Xgz7qByiS9U7k7/NZsPWrVtRVlaW5hbRWHKxT4CgzMLrR+b6xS9+gf379+Pll1+2Hpg8nq8X7MGilAqHw2htbbX29+7di/Ly8jS2iMYaPgGChsPrR+Z6/PHHsXfvXrz00kvIysqyysfz9YITjVJKffbZZ6isrISu65BSorS0FL/+9a/Hxf8MlFoPPPAAXnjhBTQ2NiI3NxderxeffPIJgJGfFEET33DnBq8fmam+vh5FRUUoLS2Fz+cDADgcDrz99tsAxu/1ggGLiIiIKMV4i5CIiIgoxRiwiIiIiFKMAYuIiIgoxRiwiIiIiFKMAYuIiIgoxRiwiIiIiFKMAYuIiIgoxRiwiIgGeOSRR+D1etPdDCIaxxiwiIiIiFKMAYuIiIgoxRiwiGhMePPNN1FRUQGPx4NAIIBvfvObaGpqAgDU1NRACIHdu3djy5YtCAQCCAaD+OEPf4h4PJ70PidPnsSaNWvg9Xrh9/vx9a9/3XoGYi/DMPD4449j9uzZcDgcyM/Px4YNG9DW1pZU74MPPsCSJUvgdrtx7bXX4m9/+9vofglENGEwYBFR2r355ptYsWIFAoEAnn32Wfzud7/DsWPHcOeddybVe/jhh2EYBp577jn8+Mc/xhNPPIHt27dbx+vq6rB06VKcOXMGu3fvxu9//3tUV1dj6dKlOHv2rFXv+9//Ph566CHccccd+NOf/oSnnnoKPp8PnZ2dVp1YLIZ7770XmzZtwoEDB5Cbm4vKyko0NzeP/hdCROOfJCJKs2XLlsnFixdLwzCsspMnT0ohhDx48KD8/PPPJQC5dOnSpNdt375dut1uGQqFpJRSbtu2TbrdbtnU1GTVqampkTabTe7YsUNKKeWpU6ekEELu3Llz2Pbs2LFDApAHDx60yk6fPi0ByGeeeSYVH5mIJjj2YBFRWkUiEbz++uvYsGEDdF1HPB5HPB7HzJkzUVBQgGPHjll177rrrqTXrl+/HpFIBB9++CEA4OjRo6ioqEBeXp5V56qrrsLixYtx9OhRAMArr7wCKSW2bNkyYrsURcEtt9xi7U+fPh12ux319fWX/ZmJaOJjwCKitGppaYGu69i2bRtsNlvS0tDQgLq6OqvupEmTkl7bu//ll19a75Wfnz/oz8jPz0coFAIANDc3Q9O0Qe81kMvlgt1uTyqz2Wzo6uq6+A9JRBlHS3cDiCizZWVlQQiBhx9+GOvWrRt0PDc319ruHfQ+cL+goAAAEAwGcebMmUHv0djYiGAwCADIyclBPB5HU1PTeUMWEdGlYg8WEaWVx+PBokWL8PHHH2P+/PmDlpKSEqvugQMHkl67f/9+uN1uzJ07FwCwZMkSHDp0KGkgel1dHd544w0sXboUAFBRUQEhBHbt2jX6H46IMhZ7sIgo7X7+85+joqIC3/jGN3DPPfcgOzsb9fX1eOmll7B582YrZH366afYvHkz7rnnHhw/fhw/+9nPsHXrVmRnZwMAtm3bhl27duHWW2/FT37yE+i6jh07diAYDOKBBx4AAJSVleG73/0utm/fjlAohFWrViESieDgwYN45JFHMGXKlHR9DUQ0gTBgEVHaLV68GK+99hp27NiBzZs3o6enB1OnTsWqVaswffp0a66rxx57DK+++io2bNgAVVXxve99D4899pj1PkVFRaiqqsKPfvQj3HfffVAUBStXrsQvf/nLpIHvTz75JKZNm4ann34av/rVr5CTk4Ply5fD5/Nd8c9ORBOTkFLKdDeCiGgkNTU1mDZtGp5//nncfffd6W4OEdF5cQwWERERUYoxYBERERGlGG8REhEREaUYe7CIiIiIUowBi4iIiCjF/j92m6g5IMsxcQAAAABJRU5ErkJggg==" - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "plot(losses,\n", " title=\"Cross Entropy\",\n", " xlab = \"epoch\",\n", " label=\"out-of-sample\")\n", "plot!(training_losses, label=\"training\")" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "Retrieving a snapshot for a prediction:" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Machine{ImageClassifier{MyConvBuilder,…},…} @200 trained 1 time; caches data\n", - " args: \n" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } ], - "source": [ - "mach2 = machine(joinpath(DIR, \"mnist_machine5.jlso\"))" - ] + "metadata": {} }, { + "outputs": [], "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "3-element CategoricalArrays.CategoricalArray{Int64,1,UInt32}:\n", - " 2\n", - " 7\n", - " 3" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ + "mach2 = machine(joinpath(DIR, \"mnist_machine5.jlso\"))\n", "predict_mode(mach2, images[501:503])" - ] + ], + "metadata": {}, + "execution_count": null }, { "cell_type": "markdown", - "metadata": {}, "source": [ "---\n", "\n", "*This notebook was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).*" - ] + ], + "metadata": {} } ], + "nbformat_minor": 3, "metadata": { - "kernelspec": { - "display_name": "Julia 1.6.0", - "language": "julia", - "name": "julia-1.6" - }, "language_info": { "file_extension": ".jl", "mimetype": "application/julia", "name": "julia", "version": "1.6.0" + }, + "kernelspec": { + "name": "julia-1.6", + "display_name": "Julia 1.6.0", + "language": "julia" } }, - "nbformat": 4, - "nbformat_minor": 3 + "nbformat": 4 } diff --git a/examples/mnist/mnist.jl b/examples/mnist/mnist.jl index 3d7bffae..b91ae03d 100644 --- a/examples/mnist/mnist.jl +++ b/examples/mnist/mnist.jl @@ -5,6 +5,8 @@ const DIR = @__DIR__ Pkg.activate(DIR) Pkg.instantiate() +# **Julia version** is assumed to be 1.6.* + using MLJ using Flux import MLJFlux diff --git a/examples/mnist/mnist.md b/examples/mnist/mnist.md index 2f1c06a1..fc284eb8 100644 --- a/examples/mnist/mnist.md +++ b/examples/mnist/mnist.md @@ -9,7 +9,11 @@ using Pkg const DIR = @__DIR__ Pkg.activate(DIR) Pkg.instantiate() +``` + +**Julia version** is assumed to be 1.6.* +```@example mnist using MLJ using Flux import MLJFlux diff --git a/src/classifier.jl b/src/classifier.jl index acc6e2ca..3a124242 100644 --- a/src/classifier.jl +++ b/src/classifier.jl @@ -113,7 +113,9 @@ function MLJModelInterface.update(model::NeuralNetworkClassifier, # we only get to keep the optimiser "state" carried over from # previous training if we're doing a warm restart and the user has not # changed the optimiser hyper-parameter: - if !keep_chain || model.optimiser != old_model.optimiser + if !keep_chain || + !MLJModelInterface._equal_to_depth_one(model.optimiser, + old_model.optimiser) optimiser = deepcopy(model.optimiser) end diff --git a/src/common.jl b/src/common.jl index 44383786..fa10c391 100644 --- a/src/common.jl +++ b/src/common.jl @@ -3,6 +3,16 @@ MLJFluxModel = Union{NeuralNetworkRegressor, NeuralNetworkClassifier, ImageClassifier} + +# # EQUALITY + +# to address #124 and #129: +MLJModelInterface.deep_properties(::Type{<:MLJFluxModel}) = + (:optimiser, :builder) + + +# # CLEAN METHOD + function MLJModelInterface.clean!(model::MLJFluxModel) warning = "" if model.lambda < 0 diff --git a/src/core.jl b/src/core.jl index ae7b4b0a..bef4f2f4 100644 --- a/src/core.jl +++ b/src/core.jl @@ -1,39 +1,29 @@ ## EXPOSE OPTIMISERS TO MLJ (for eg, tuning) -# Here we: (i) Make the optimiser structs "transparent" so that their -# field values are exposed by calls to MLJ.params; and (ii) Overload -# `==` for optimisers, so that we can detect when their parameters -# remain unchanged on calls to MLJModelInterface.update methods. - -# We define optimisers of to be `==` if: (i) They have identical type -# AND (ii) their defined field values are `==`. (Note that our `fit` -# methods will only use deep copies of optimisers specified as -# hyperparameters because some fields of `optimisers` carry "state" -# information which is mutated during chain updates.) - -for opt in (:Descent, :Momentum, :Nesterov, :RMSProp, :ADAM, :AdaMax, - :ADAGrad, :ADADelta, :AMSGrad, :NADAM, :Optimiser, - :InvDecay, :ExpDecay, :WeightDecay) +# Here we make the optimiser structs "transparent" so that their +# field values are exposed by calls to MLJ.params + +for opt in (:Descent, + :Momentum, + :Nesterov, + :RMSProp, + :ADAM, + :RADAM, + :AdaMax, + :OADAM, + :ADAGrad, + :ADADelta, + :AMSGrad, + :NADAM, + :AdaBelief, + :Optimiser, + :InvDecay, :ExpDecay, :WeightDecay, + :ClipValue, + :ClipNorm) # last updated: Flux.jl 0.12.3 @eval begin - - # TODO: Uncomment next line when - # https://github.com/alan-turing-institute/MLJModelInterface.jl/issues/28 - # is resolved: - - # MLJModelInterface.istransparent(m::Flux.$opt) = true - - function ==(m1::Flux.$opt, m2::Flux.$opt) - same_values = true - for fld in fieldnames(Flux.$opt) - same_values = same_values && - getfield(m1, fld) == getfield(m2, fld) - end - return same_values - end - + MLJModelInterface.istransparent(m::Flux.$opt) = true end - end @@ -322,5 +312,3 @@ function collate(model, X, y) ymatrix = reformat(y) return [_get(Xmatrix, b) for b in row_batches], [_get(ymatrix, b) for b in row_batches] end - - diff --git a/src/image.jl b/src/image.jl index 0154fe0d..42884730 100644 --- a/src/image.jl +++ b/src/image.jl @@ -126,7 +126,9 @@ function MLJModelInterface.update(model::ImageClassifier, # we only get to keep the optimiser "state" carried over from # previous training if we're doing a warm restart and the user has not # changed the optimiser hyper-parameter: - if !keep_chain || model.optimiser != old_model.optimiser + if !keep_chain || + !MLJModelInterface._equal_to_depth_one(model.optimiser, + old_model.optimiser) optimiser = deepcopy(model.optimiser) end @@ -157,7 +159,7 @@ MLJModelInterface.fitted_params(::ImageClassifier, fitresult) = (chain=fitresult[1],) MLJModelInterface.metadata_model(ImageClassifier, - input=AbstractVector{<:MLJModelInterface.GrayImage}, + input=AbstractVector{<:MLJModelInterface.Image}, target=AbstractVector{<:Multiclass}, path="MLJFlux.ImageClassifier", descr="A neural network model for making probabilistic predictions of a `GrayImage` target, diff --git a/src/regressor.jl b/src/regressor.jl index 2e370b96..9b992086 100644 --- a/src/regressor.jl +++ b/src/regressor.jl @@ -151,7 +151,9 @@ function MLJModelInterface.update(model::Regressor, # we only get to keep the optimiser "state" carried over from # previous training if we're doing a warm restart and the user has not # changed the optimiser hyper-parameter: - if !keep_chain || model.optimiser != old_model.optimiser + if !keep_chain || + !MLJModelInterface._equal_to_depth_one(model.optimiser, + old_model.optimiser) optimiser = deepcopy(model.optimiser) end diff --git a/test/common.jl b/test/common.jl index 12989e60..6b15aca4 100644 --- a/test/common.jl +++ b/test/common.jl @@ -1,5 +1,17 @@ ModelType = MLJFlux.NeuralNetworkRegressor +@testset "equality" begin + model = MLJFlux.ImageClassifier() + clone = deepcopy(model) + @test model == clone + clone.optimiser.eta *= 10 + @test model != clone + + clone = deepcopy(model) + clone.builder.dropout *= 0.5 + @test clone != model +end + @testset "clean!" begin model = @test_logs (:warn, r"`lambda") begin ModelType(lambda = -1) @@ -42,3 +54,4 @@ end @test losses == MLJBase.report(mach).training_losses[2:end] @test length(losses) == 10 end + diff --git a/test/core.jl b/test/core.jl index cdcd84aa..e93e73d9 100644 --- a/test/core.jl +++ b/test/core.jl @@ -1,10 +1,6 @@ Random.seed!(123) -@testset "optimiser equality" begin - @test Flux.Momentum() == Flux.Momentum() - @test Flux.Momentum(0.1) != Flux.Momentum(0.2) - @test Flux.ADAM(0.1) != Flux.ADAM(0.2) -end +@test MLJFlux.MLJModelInterface.istransparent(Flux.ADAM(0.1)) @testset "nrows" begin Xmatrix = rand(10, 3) @@ -39,13 +35,13 @@ end y = MLJBase.table(ymatrix) # a rowaccess table model = MLJFlux.NeuralNetworkRegressor() model.batch_size= 3 - @test MLJFlux.collate(model, X, y) == + @test MLJFlux.collate(model, X, y) == ([Xmatrix'[:,1:3], Xmatrix'[:,4:6], Xmatrix'[:,7:9], Xmatrix'[:,10:10]], [ymatrix'[:,1:3], ymatrix'[:,4:6], ymatrix'[:,7:9], ymatrix'[:,10:10]]) y = Tables.columntable(y) # try a columnaccess table @test MLJFlux.collate(model, X, y) == - ([Xmatrix'[:,1:3], Xmatrix'[:,4:6], Xmatrix'[:,7:9], Xmatrix'[:,10:10]], + ([Xmatrix'[:,1:3], Xmatrix'[:,4:6], Xmatrix'[:,7:9], Xmatrix'[:,10:10]], [ymatrix'[:,1:3], ymatrix'[:,4:6], ymatrix'[:,7:9], ymatrix'[:,10:10]]) # ImageClassifier @@ -73,7 +69,7 @@ data = [(Xmatrix'[:,1:20], y[1:20]), (Xmatrix'[:,41:60], y[41:60]), (Xmatrix'[:,61:80], y[61:80]), (Xmatrix'[:, 81:100], y[81:100])] - + data = ([Xmatrix'[:,1:20], Xmatrix'[:,21:40], Xmatrix'[:,41:60], Xmatrix'[:,61:80], Xmatrix'[:,81:100]], [y[1:20], y[21:40], y[41:60], y[61:80], y[81:100]]) @@ -103,9 +99,9 @@ epochs = 10 _chain_yes_drop, history = MLJFlux.fit!(chain_yes_drop, Flux.Optimise.ADAM(0.001), Flux.mse, epochs, 0, 0, 0, accel, data[1], data[2]) - + println() - + Random.seed!(123) _chain_no_drop, history = MLJFlux.fit!(chain_no_drop,