platform-demo-scripts/notebooks/Check model performance depending on station-fixed window.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "05600e0a-8b4b-42ab-96d5-9d6eb2c72102",
   "metadata": {},
   "source": [
    "# Check model performance depending on station\n",
    "\n",
    "#### Samples generated with fixed window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1732f721-6b13-4fb3-8755-dc63cb255285",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mkmilian\u001b[0m (\u001b[33mepos\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /Users/krystynamilian/.netrc\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from obspy.core.event import read_events\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import seisbench.models as sbm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "import seisbench.data as sbd\n",
    "import seisbench.generate as sbg\n",
    "import seisbench.models as sbm\n",
    "from seisbench.util import worker_seeding\n",
    "import numpy as np\n",
    "from torch.utils.data import DataLoader\n",
    "from pathlib import Path\n",
    "import wandb\n",
    "import os\n",
    "import sys\n",
    "\n",
    "from pathlib import Path\n",
    "project_path = str(Path.cwd().parent)\n",
    "sys.path.append(project_path)\n",
    "from scripts import train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1a322214-ba56-4651-966d-f9afdcfecbc5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "wandb version 0.15.4 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.15.3"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/Users/krystynamilian/Documents/praca/Cyfronet/epos/ai/repo/demo_scripts/notebooks/wandb/run-20230704_110541-ir19n1xv</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/epos/demo_scripts-notebooks/runs/ir19n1xv' target=\"_blank\">eager-forest-7</a></strong> to <a href='https://wandb.ai/epos/demo_scripts-notebooks' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/epos/demo_scripts-notebooks' target=\"_blank\">https://wandb.ai/epos/demo_scripts-notebooks</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/epos/demo_scripts-notebooks/runs/ir19n1xv' target=\"_blank\">https://wandb.ai/epos/demo_scripts-notebooks/runs/ir19n1xv</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m:   1 of 1 files downloaded.  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "PhaseNet(\n",
       "  (inc): Conv1d(3, 8, kernel_size=(7,), stride=(1,), padding=same)\n",
       "  (in_bn): BatchNorm1d(8, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (down_branch): ModuleList(\n",
       "    (0): ModuleList(\n",
       "      (0): Conv1d(8, 8, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (1): BatchNorm1d(8, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(8, 8, kernel_size=(7,), stride=(4,), padding=(3,), bias=False)\n",
       "      (3): BatchNorm1d(8, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (1): ModuleList(\n",
       "      (0): Conv1d(8, 16, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (1): BatchNorm1d(16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(16, 16, kernel_size=(7,), stride=(4,), bias=False)\n",
       "      (3): BatchNorm1d(16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (2): ModuleList(\n",
       "      (0): Conv1d(16, 32, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (1): BatchNorm1d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(32, 32, kernel_size=(7,), stride=(4,), bias=False)\n",
       "      (3): BatchNorm1d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (3): ModuleList(\n",
       "      (0): Conv1d(32, 64, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (1): BatchNorm1d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(64, 64, kernel_size=(7,), stride=(4,), bias=False)\n",
       "      (3): BatchNorm1d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (4): ModuleList(\n",
       "      (0): Conv1d(64, 128, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (1): BatchNorm1d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2-3): 2 x None\n",
       "    )\n",
       "  )\n",
       "  (up_branch): ModuleList(\n",
       "    (0): ModuleList(\n",
       "      (0): ConvTranspose1d(128, 64, kernel_size=(7,), stride=(4,), bias=False)\n",
       "      (1): BatchNorm1d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(128, 64, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (3): BatchNorm1d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (1): ModuleList(\n",
       "      (0): ConvTranspose1d(64, 32, kernel_size=(7,), stride=(4,), bias=False)\n",
       "      (1): BatchNorm1d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(64, 32, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (3): BatchNorm1d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (2): ModuleList(\n",
       "      (0): ConvTranspose1d(32, 16, kernel_size=(7,), stride=(4,), bias=False)\n",
       "      (1): BatchNorm1d(16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(32, 16, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (3): BatchNorm1d(16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (3): ModuleList(\n",
       "      (0): ConvTranspose1d(16, 8, kernel_size=(7,), stride=(4,), bias=False)\n",
       "      (1): BatchNorm1d(8, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(16, 8, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (3): BatchNorm1d(8, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (out): Conv1d(8, 2, kernel_size=(1,), stride=(1,), padding=same)\n",
       "  (softmax): Softmax(dim=1)\n",
       ")"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = train.load_model()\n",
    "\n",
    "run = wandb.init()\n",
    "artifact = run.use_artifact('epos/training_seisbench_models_on_igf_data/model:v113', type='model')\n",
    "artifact_dir = artifact.download()\n",
    "fname = artifact_dir + \"/\" + os.listdir(artifact_dir)[0]\n",
    "\n",
    "model.load_state_dict(torch.load(fname))\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "75cadd4b-7a0e-44cd-acc6-fef26f2cf551",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../../../data/igf/seisbench_format'\n",
    "sampling_rate = 100\n",
    "data = sbd.WaveformDataset(data_path, sampling_rate=sampling_rate)\n",
    "data.filter(data.metadata.trace_Pg_arrival_sample.notna())\n",
    "\n",
    "pick_mae = train.PickMAE(sampling_rate)\n",
    "splits = ['train', 'dev', 'test']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccf454fe-e55a-47bc-882a-3e39395147a5",
   "metadata": {},
   "source": [
    "### Calculate overall model performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4c7b4137-4599-4599-9e3e-064145bfccb4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Model resutls for train set\n",
      "\n",
      "Fixed window\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.025157\n",
      "Test avg mae: 0.047488\n",
      "\n",
      "\n",
      "\n",
      "Model resutls for dev set\n",
      "\n",
      "Fixed window\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.025309\n",
      "Test avg mae: 0.051242\n",
      "\n",
      "\n",
      "\n",
      "Model resutls for test set\n",
      "\n",
      "Fixed window\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.025555\n",
      "Test avg mae: 0.047198\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for split in splits: \n",
    "    print(f\"\\n\\nModel resutls for {split} set\")\n",
    "    print(\"\\nFixed window\")\n",
    "    # the results depend on random selection of a window around a pick\n",
    "    gen = train.get_data_generator(split=split, station=None, sampling_rate=sampling_rate, path=data_path, window='fixed')\n",
    "\n",
    "    data_loader = DataLoader(gen, batch_size=256, shuffle=False, num_workers=0,\n",
    "                                    worker_init_fn=worker_seeding)\n",
    "        \n",
    "    test_loss, test_mae = train.test_one_epoch(model, data_loader, pick_mae, wandb_log=False)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a361bc6a-2dac-4e27-9417-3abe482161a5",
   "metadata": {},
   "source": [
    "## Check # frames per station in each set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b23b71b7-ba71-4e86-8978-5a70d4fbbdfb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: title={'center': 'Frames per station'}, xlabel='station_code'>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABMsAAAFhCAYAAABu0i2NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACGuUlEQVR4nOzdd1hT598G8Dthb8EB4gIV3Ksu0FZQqDgr1lEpVdFWW6t2+ENFrQv3XtXaOhDr1rptHVXBAVLFgeLAgaNVcAKCynzeP4TzGghDi5wTe3+uK9dlznOS3IkhOfmeZ6iEEAJEREREREREREQEtdwBiIiIiIiIiIiIlILFMiIiIiIiIiIiomwslhEREREREREREWVjsYyIiIiIiIiIiCgbi2VERERERERERETZWCwjIiIiIiIiIiLKxmIZERERERERERFRNhbLiIiIiIiIiIiIsrFYRkRERERERERElI3FMiIiIiLSSQ4ODvDz85M7BhEREb1jWCwjIiIinbJq1SqoVCqtl4CAALnj0SuePXuGCRMmICQk5I3vIywsDBMmTEBCQkKx5SIiIiIqiL7cAYiIiIjeRGBgIBwdHTW21a1bV6Y0pM2zZ88wceJEAIC7u/sb3UdYWBgmTpwIPz8/lCpVSqPtypUrUKt57peIiIiKF4tlREREpJPat2+PJk2aFGnfFy9ewNDQkIWV15SVlYW0tDQYGxvLHUUrIyMjuSMQERHRO4hHjERERPROCQkJgUqlwoYNG/DDDz+gQoUKMDU1RVJSEh4/fgx/f3/Uq1cP5ubmsLS0RPv27XHu3Dmt97Fp0yZMnDgRFSpUgIWFBbp3747ExESkpqbiu+++Q7ly5WBubo5+/fohNTU1T5Y1a9agcePGMDExgY2NDXr16oU7d+5o7HP16lV069YNdnZ2MDY2RsWKFdGrVy8kJiYW+Dzd3d1Rt25dREZGokWLFjAxMYGjoyOWLl2aZ9/U1FSMHz8e1atXh5GRESpVqoQRI0bkyaxSqTBkyBCsXbsWderUgZGREfbu3ZtvhlOnTsHLywtlypSRHr9///4AgJs3b6Js2bIAgIkTJ0pDZSdMmAAAiIqKgp+fH6pWrQpjY2PY2dmhf//+ePTokXT/EyZMwPDhwwEAjo6O0n3cvHkTgPY5y27cuIEePXrAxsYGpqamcHFxwZ49ezT2efX/d8qUKahYsSKMjY3h4eGBa9euFfi6ExER0buPPcuIiIhIJyUmJuLhw4ca28qUKSP9e9KkSTA0NIS/vz9SU1NhaGiIixcvYvv27ejRowccHR0RHx+Pn3/+GW5ubrh48SLs7e017m/atGkwMTFBQEAArl27hkWLFsHAwABqtRpPnjzBhAkTcOLECaxatQqOjo4YN26cdNspU6Zg7Nix6NmzJ7744gs8ePAAixYtQqtWrXDmzBmUKlUKaWlp8PLyQmpqKoYOHQo7Ozv8888/2L17NxISEmBlZVXga/DkyRN06NABPXv2hI+PDzZt2oRBgwbB0NBQKlplZWXho48+wrFjxzBw4EDUqlUL58+fx7x58xATE4Pt27dr3OehQ4ewadMmDBkyBGXKlIGDg4PWx75//z7atm2LsmXLIiAgAKVKlcLNmzexdetWAEDZsmXx008/YdCgQejatSs+/vhjAED9+vUBAAcOHMCNGzfQr18/2NnZITo6Gr/88guio6Nx4sQJqFQqfPzxx4iJicH69esxb9486f83pwiXW3x8PFq0aIFnz57hm2++QenSpREcHIyPPvoIW7ZsQdeuXTX2nz59OtRqNfz9/ZGYmIiZM2fC19cXERERBb7uRERE9I4TRERERDokKChIANB6EUKIw4cPCwCiatWq4tmzZxq3ffHihcjMzNTYFhsbK4yMjERgYKC0Lec+6tatK9LS0qTtPj4+QqVSifbt22vch6urq6hSpYp0/ebNm0JPT09MmTJFY7/z588LfX19afuZM2cEALF58+bXfh3c3NwEADFnzhxpW2pqqmjYsKEoV66clPvXX38VarVaHD16VOP2S5cuFQDE8ePHpW0AhFqtFtHR0YU+/rZt2wQAcfLkyXz3efDggQAgxo8fn6ct9/+NEEKsX79eABBHjhyRts2aNUsAELGxsXn2r1Kliujbt690/bvvvhMANJ7r06dPhaOjo3BwcJD+73P+f2vVqiVSU1OlfRcsWCAAiPPnzxf01ImIiOgdx2GYREREpJMWL16MAwcOaFxe1bdvX5iYmGhsMzIykuYty8zMxKNHj2Bubo4aNWrg9OnTeR6jT58+MDAwkK43b94cQgip19ar2+/cuYOMjAwAwNatW5GVlYWePXvi4cOH0sXOzg5OTk44fPgwAEg9x/bt24dnz5699mugr6+PL7/8UrpuaGiIL7/8Evfv30dkZCQAYPPmzahVqxZq1qypkaVNmzYAIGXJ4ebmhtq1axf62DmT7e/evRvp6emvnf3V/5sXL17g4cOHcHFxAQCt/xdF8fvvv6NZs2Z4//33pW3m5uYYOHAgbt68iYsXL2rs369fPxgaGkrXP/jgAwAvh3ISERHRfxeLZURERKSTmjVrBk9PT43Lq3KvlAm8HJI4b948ODk5wcjICGXKlEHZsmURFRWldY6wypUra1zPKW5VqlQpz/asrCzpPq5evQohBJycnFC2bFmNy6VLl3D//n0p47Bhw7B8+XKUKVMGXl5eWLx4caHzleWwt7eHmZmZxjZnZ2cAkOb1unr1KqKjo/PkyNkvJ0tBr5s2bm5u6NatGyZOnIgyZcqgS5cuCAoK0jp3mzaPHz/Gt99+C1tbW5iYmKBs2bLSYxf1+ed269Yt1KhRI8/2WrVqSe2vyv3/a21tDeDl8FYiIiL67+KcZURERPROyt2rDACmTp2KsWPHon///pg0aRJsbGygVqvx3XffISsrK8/+enp6Wu87v+1CCAAvi3IqlQp//PGH1n3Nzc2lf8+ZMwd+fn7YsWMH9u/fj2+++QbTpk3DiRMnULFixSI914JkZWWhXr16mDt3rtb23IU/ba+bNiqVClu2bMGJEyewa9cu7Nu3D/3798ecOXNw4sQJjeeoTc+ePREWFobhw4ejYcOGMDc3R1ZWFtq1a6f1/+JtKOz/kYiIiP6bWCwjIiKi/4wtW7agdevWWLFihcb2hIQEjcUB/q1q1apBCAFHR0epB1dB6tWrh3r16uGHH35AWFgYWrZsiaVLl2Ly5MkF3u7u3btISUnR6F0WExMDANLE/NWqVcO5c+fg4eEBlUr15k8qHy4uLnBxccGUKVOwbt06+Pr6YsOGDfjiiy/yfbwnT57g4MGDmDhxosaiCFevXs2z7+tkrlKlCq5cuZJn++XLl6V2IiIiosJwGCYRERH9Z+jp6eXpNbR582b8888/xfo4H3/8MfT09DBx4sQ8jyeEwKNHjwAASUlJ0jxnOerVqwe1Wl2k4YwZGRn4+eefpetpaWn4+eefUbZsWTRu3BjAyx5c//zzD5YtW5bn9s+fP0dKSsprPz/gZcEr93Nr2LAhAEjZTU1NAbwsRr4qp0dX7tvPnz8/z+PkFAJz34c2HTp0wF9//YXw8HBpW0pKCn755Rc4ODgUaS42IiIiIvYsIyIiov+MTp06ITAwEP369UOLFi1w/vx5rF27FlWrVi3Wx6lWrRomT56MUaNG4ebNm/D29oaFhQViY2Oxbds2DBw4EP7+/jh06BCGDBmCHj16wNnZGRkZGfj111+hp6eHbt26Ffo49vb2mDFjBm7evAlnZ2ds3LgRZ8+exS+//CItTNC7d29s2rQJX331FQ4fPoyWLVsiMzMTly9fxqZNm7Bv3z40adLktZ9jcHAwlixZgq5du6JatWp4+vQpli1bBktLS3To0AHAyyGdtWvXxsaNG+Hs7AwbGxvUrVsXdevWRatWrTBz5kykp6ejQoUK2L9/P2JjY/M8Tk7Rb8yYMejVqxcMDAzQuXPnPHO1AUBAQADWr1+P9u3b45tvvoGNjQ2Cg4MRGxuL3377TVrcgYiIiKggLJYRERHRf8bo0aORkpKCdevWYePGjXjvvfewZ88eBAQEFPtjBQQ
      "text/plain": [
       "<Figure size 1500x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "frames_per_station = []\n",
    "for split in splits: \n",
    "    frames_per_station.append(data.get_split(split).metadata.groupby('station_code').count()['index'])\n",
    "    \n",
    "frames_per_station = pd.DataFrame(frames_per_station, index=splits).transpose()\n",
    "frames_per_station.plot(kind='bar', figsize=(15,3), title='Frames per station')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acd81c2b-318f-45ea-b639-bf1ed26b0c10",
   "metadata": {},
   "source": [
    "## Calculate MAE per station for train/dev/test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "66992bf3-c49a-4885-8ab1-5b372ad65202",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train\n",
      "BRDW\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.029592\n",
      "Test avg mae: 0.089062\n",
      "\n",
      "GROD\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.025606\n",
      "Test avg mae: 0.057617\n",
      "\n",
      "GUZI\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.024516\n",
      "Test avg mae: 0.040501\n",
      "\n",
      "JEDR\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.024908\n",
      "Test avg mae: 0.044932\n",
      "\n",
      "MOSK2\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.024886\n",
      "Test avg mae: 0.040648\n",
      "\n",
      "NWLU\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.024519\n",
      "Test avg mae: 0.032965\n",
      "\n",
      "PCHB\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.024587\n",
      "Test avg mae: 0.042854\n",
      "\n",
      "PPOL\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.026397\n",
      "Test avg mae: 0.076008\n",
      "\n",
      "RUDN\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.025373\n",
      "Test avg mae: 0.047411\n",
      "\n",
      "RYNR\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.025934\n",
      "Test avg mae: 0.066802\n",
      "\n",
      "RZEC\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.023816\n",
      "Test avg mae: 0.029310\n",
      "\n",
      "SGOR\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.024461\n",
      "Test avg mae: 0.034385\n",
      "\n",
      "TRBC2\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.025840\n",
      "Test avg mae: 0.046694\n",
      "\n",
      "TRN2\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.025408\n",
      "Test avg mae: 0.051002\n",
      "\n",
      "TRZS\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.025081\n",
      "Test avg mae: 0.043014\n",
      "\n",
      "ZMST\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.025119\n",
      "Test avg mae: 0.049605\n",
      "\n",
      "LUBW\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.029460\n",
      "Test avg mae: 0.095455\n",
      "\n",
      "DWOL\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.024191\n",
      "Test avg mae: 0.026768\n",
      "\n",
      "LUBZ\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.032006\n",
      "Test avg mae: 0.180000\n",
      "\n",
      "ZUKW2\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.024766\n",
      "Test avg mae: 0.032847\n",
      "\n",
      "DABR\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.024297\n",
      "Test avg mae: 0.030349\n",
      "\n",
      "PEKW2\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.025126\n",
      "Test avg mae: 0.044390\n",
      "\n",
      "KRZY\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.025487\n",
      "Test avg mae: 0.060000\n",
      "\n",
      "OBIS\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.024071\n",
      "Test avg mae: 0.028000\n",
      "\n",
      "KAZI\n",
      "train (12444, 17) 100\n",
      "Test avg loss: 0.023955\n",
      "Test avg mae: 0.025048\n",
      "\n",
      "KWLC\n",
      "train (12444, 17) 100\n",
      "division by zero\n",
      "dev\n",
      "BRDW\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.029170\n",
      "Test avg mae: 0.093000\n",
      "\n",
      "GROD\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.025056\n",
      "Test avg mae: 0.045482\n",
      "\n",
      "GUZI\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.024413\n",
      "Test avg mae: 0.039008\n",
      "\n",
      "JEDR\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.024724\n",
      "Test avg mae: 0.017778\n",
      "\n",
      "MOSK2\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.024798\n",
      "Test avg mae: 0.039188\n",
      "\n",
      "NWLU\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.025681\n",
      "Test avg mae: 0.037319\n",
      "\n",
      "PCHB\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.024584\n",
      "Test avg mae: 0.044390\n",
      "\n",
      "PPOL\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.025954\n",
      "Test avg mae: 0.074627\n",
      "\n",
      "RUDN\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.025528\n",
      "Test avg mae: 0.049302\n",
      "\n",
      "RYNR\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.026649\n",
      "Test avg mae: 0.071854\n",
      "\n",
      "RZEC\n",
      "dev (2773, 17) 100\n",
      "division by zero\n",
      "SGOR\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.025600\n",
      "Test avg mae: 0.083871\n",
      "\n",
      "TRBC2\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.025220\n",
      "Test avg mae: 0.032712\n",
      "\n",
      "TRN2\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.024823\n",
      "Test avg mae: 0.039834\n",
      "\n",
      "TRZS\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.026505\n",
      "Test avg mae: 0.093793\n",
      "\n",
      "ZMST\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.024234\n",
      "Test avg mae: 0.030628\n",
      "\n",
      "LUBW\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.029821\n",
      "Test avg mae: 0.129583\n",
      "\n",
      "DWOL\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.024637\n",
      "Test avg mae: 0.048252\n",
      "\n",
      "LUBZ\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.023594\n",
      "Test avg mae: 0.010000\n",
      "\n",
      "ZUKW2\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.024708\n",
      "Test avg mae: 0.035825\n",
      "\n",
      "DABR\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.024622\n",
      "Test avg mae: 0.036894\n",
      "\n",
      "PEKW2\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.026771\n",
      "Test avg mae: 0.046392\n",
      "\n",
      "KRZY\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.038841\n",
      "Test avg mae: 0.099091\n",
      "\n",
      "OBIS\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.025738\n",
      "Test avg mae: 0.112784\n",
      "\n",
      "KAZI\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.024498\n",
      "Test avg mae: 0.029636\n",
      "\n",
      "KWLC\n",
      "dev (2773, 17) 100\n",
      "Test avg loss: 0.023677\n",
      "Test avg mae: 0.030000\n",
      "\n",
      "test\n",
      "BRDW\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.026271\n",
      "Test avg mae: 0.062836\n",
      "\n",
      "GROD\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.025284\n",
      "Test avg mae: 0.057771\n",
      "\n",
      "GUZI\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.025403\n",
      "Test avg mae: 0.050159\n",
      "\n",
      "JEDR\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.027117\n",
      "Test avg mae: 0.039194\n",
      "\n",
      "MOSK2\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.024491\n",
      "Test avg mae: 0.035829\n",
      "\n",
      "NWLU\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.024571\n",
      "Test avg mae: 0.030738\n",
      "\n",
      "PCHB\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.024751\n",
      "Test avg mae: 0.039241\n",
      "\n",
      "PPOL\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.025289\n",
      "Test avg mae: 0.047288\n",
      "\n",
      "RUDN\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.025869\n",
      "Test avg mae: 0.056053\n",
      "\n",
      "RYNR\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.027814\n",
      "Test avg mae: 0.101556\n",
      "\n",
      "RZEC\n",
      "test (2785, 17) 100\n",
      "division by zero\n",
      "SGOR\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.025836\n",
      "Test avg mae: 0.043481\n",
      "\n",
      "TRBC2\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.024740\n",
      "Test avg mae: 0.031455\n",
      "\n",
      "TRN2\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.027810\n",
      "Test avg mae: 0.063457\n",
      "\n",
      "TRZS\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.024783\n",
      "Test avg mae: 0.036907\n",
      "\n",
      "ZMST\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.025570\n",
      "Test avg mae: 0.050635\n",
      "\n",
      "LUBW\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.024866\n",
      "Test avg mae: 0.038000\n",
      "\n",
      "DWOL\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.024782\n",
      "Test avg mae: 0.035500\n",
      "\n",
      "LUBZ\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.023713\n",
      "Test avg mae: 0.020000\n",
      "\n",
      "ZUKW2\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.024829\n",
      "Test avg mae: 0.034684\n",
      "\n",
      "DABR\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.025824\n",
      "Test avg mae: 0.040625\n",
      "\n",
      "PEKW2\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.024243\n",
      "Test avg mae: 0.036092\n",
      "\n",
      "KRZY\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.025173\n",
      "Test avg mae: 0.060000\n",
      "\n",
      "OBIS\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.026633\n",
      "Test avg mae: 0.056509\n",
      "\n",
      "KAZI\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.025799\n",
      "Test avg mae: 0.038860\n",
      "\n",
      "KWLC\n",
      "test (2785, 17) 100\n",
      "Test avg loss: 0.025239\n",
      "Test avg mae: 0.052885\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stations = data.metadata.station_code.unique()\n",
    "\n",
    "results = []\n",
    "# highest_dev_mae = 0\n",
    "# highest_test_mae = 0\n",
    "\n",
    "for split in splits: \n",
    "    split_results = {}\n",
    "    print(split)\n",
    "    for station in stations: \n",
    "        print(station)\n",
    "        \n",
    "        split_results[station] = {}\n",
    "        gen = train.get_data_generator(split=split, station=station, sampling_rate=sampling_rate, path=data_path, window='fixed')\n",
    "        data_loader = DataLoader(gen, batch_size=256, shuffle=False, num_workers=0,\n",
    "                                worker_init_fn=worker_seeding)\n",
    "    \n",
    "        test_loss, test_mae = None, None\n",
    "        try: \n",
    "            test_loss, test_mae = train.test_one_epoch(model, data_loader, pick_mae, wandb_log=False)\n",
    "            test_mae = float(test_mae)\n",
    "            \n",
    "        except Exception as e: \n",
    "            print(e)\n",
    "    \n",
    "        split_results[station]['mae']=test_mae\n",
    "        split_results[station]['loss']=test_loss\n",
    "    results.append(split_results)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "206e6973-61bb-4f84-95a5-f910c7c7dc2b",
   "metadata": {},
   "source": [
    "#### Plot MAE per station"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f0366dc6-d1e1-44cb-b748-d0af44a77219",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABM8AAAGaCAYAAAAYbGQtAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACRK0lEQVR4nOzdd3gU5f7+8Xs3kIT00BJ66L1JRyBBogHBI4p0pYigHEAlX5CmdAxKERUQUYhyFEGwgOBBEQigFKUrvaNAQk+oCSTP7w9+2cOSbEgQsht8v65rLth5np29dzK7O/vZZ2YsxhgjAAAAAAAAAGlYnR0AAAAAAAAAcFUUzwAAAAAAAAAHKJ4BAAAAAAAADlA8AwAAAAAAABygeAYAAAAAAAA4QPEMAAAAAAAAcIDiGQAAAAAAAOAAxTMAAAAAAADAAYpnAAAAAAAAgAMUzwAAwD9eSEiIunXr5uwYDzyLxaKRI0dm++OOHDlSFosl2x/3bnXr1k0hISHOjgEAAP4/imcAAOQQn3zyiSwWiywWi37++ec07cYYFStWTBaLRa1atXJCwgfHrl27NHLkSB05csTZUbLViRMnNHLkSG3btu2ul/H99987pUCWne7Fesqp/qmvDQDAPxvFMwAAchhPT0/NnTs3zfzVq1frr7/+koeHhxNSPVh27dqlUaNG/eMKBCdOnNCoUaP+dvFs1KhR6bZdvXpVr7/++l0v+269/vrrunr16j1b3r1YTxn56KOPtHfv3vuy7L/rn/raAAD8s1E8AwAgh3n88ce1YMEC3bhxw27+3LlzVatWLQUHBzsp2d1LSUnRtWvXnB0jx7tx44aSkpKcHcMhT09P5cqVK9sfN1euXPL09Mz2x0115cqVLPXPnTs3RXAAAFwIxTMAAHKYjh076uzZs1q+fLltXlJSkhYuXKhOnTqle5+UlBRNmTJFlStXlqenp4KCgvTiiy/q/Pnzdv0WLVqkli1bqnDhwvLw8FDp0qU1ZswYJScn2/ULCwtTlSpVtGvXLjVt2lReXl4qUqSI3n777Uw9B4vFor59++rzzz9X5cqV5eHhoWXLlkmSjh8/rueff15BQUHy8PBQ5cqVNXv27DTLeP/991W5cmV5eXkpMDBQtWvXthuR5+i8UXc6/9Unn3yitm3bSpKaNm1qO1Q2JiZGkrRp0yZFREQof/78ypMnj0qWLKnnn3/+js85JCRErVq10o8//qgaNWrI09NTlSpV0tdff52m74ULF/Tqq6+qWLFi8vDwUJkyZfTWW28pJSXF1ufIkSOyWCyaOHGipkyZotKlS8vDw0O7du1ymGH58uVq1KiRAgIC5OPjo/Lly2vo0KGSpJiYGNWpU0eS1L17d9vz/uSTTyRJa9euVdu2bVW8eHF5eHioWLFi6t+/v92Irm7dumnatGmSZLv/res6vXOebd26VS1atJCfn598fHzUrFkzbdiwwa5P6iHLv/zyiyIjI1WgQAF5e3vrqaee0unTp++47tP7m6dug99++62qVKli29ZSt0NH7rSeUl8bmzdvVpMmTeTl5WVbx5l9fd2+7d76t545c6btb12nTh399ttvd3z+169f16hRo1S2bFl5enoqX758atSokd17iCTt2bNHzzzzjPLmzStPT0/Vrl1bixcvtrXf6bUBAMCDKvt/+gMAAH9LSEiIGjRooC+++EItWrSQJP33v/9VfHy8OnTooPfeey/NfV588UV98skn6t69u15++WUdPnxYU6dO1datW/XLL78od+7ckm5+Ofbx8VFkZKR8fHy0cuVKDR8+XAkJCZowYYLdMs+fP6/mzZvr6aefVrt27bRw4UINGjRIVatWteXKyMqVK/Xll1+qb9++yp8/v0JCQhQXF6f69evbChsFChTQf//7X/Xo0UMJCQl69dVXJd08rO3ll1/WM888o1deeUXXrl3Tjh07tHHjRocFxMxq0qSJXn75Zb333nsaOnSoKlasKEmqWLGiTp06pccee0wFChTQ4MGDFRAQoCNHjqRbAEvP/v371b59e7300kvq2rWroqOj1bZtWy1btkyPPvqopJujlEJDQ3X8+HG9+OKLKl68uNatW6chQ4bo5MmTmjJlit0yo6Ojde3aNfXq1UseHh7Kmzdvuo+9c+dOtWrVStWqVdPo0aPl4eGhAwcO6JdffrE9v9GjR2v48OHq1auXGjduLElq2LChJGnBggW6cuWKevfurXz58unXX3/V+++/r7/++ksLFiyQdHM7O3HihJYvX67//Oc/d1wfO3fuVOPGjeXn56fXXntNuXPn1ocffqiwsDCtXr1a9erVs+vfr18/BQYGasSIETpy5IimTJmivn37av78+Zla/7f7+eef9fXXX+vf//63fH199d5776lNmzY6duyY8uXLl+597rSeJOns2bNq0aKFOnTooGeffVZBQUGSsvb6Ss/cuXN18eJFvfjii7JYLHr77bf19NNP69ChQ7bXcHpGjhypqKgovfDCC6pbt64SEhK0adMmbdmyxbbd7dy5Uw8//LCKFCmiwYMHy9vbW19++aVat26tr776Sk899VSGrw0AAB5oBgAA5AjR0dFGkvntt9/M1KlTja+vr7ly5Yoxxpi2bduapk2bGmOMKVGihGnZsqXtfmvXrjWSzOeff263vGXLlqWZn7q8W7344ovGy8vLXLt2zTYvNDTUSDJz5syxzUtMTDTBwcGmTZs2d3wukozVajU7d+60m9+jRw9TqFAhc+bMGbv5HTp0MP7+/rZ8Tz75pKlcuXKGj9G1a1dTokSJNPNHjBhhbt8FKlGihOnatavt9oIFC4wks2rVKrt+33zzje1vkFUlSpQwksxXX31lmxcfH28KFSpkatasaZs3ZswY4+3tbfbt22d3/8GDBxs3Nzdz7NgxY4wxhw8fNpKMn5+fOXXq1B0f/5133jGSzOnTpx32+e2334wkEx0dnaYtvW0jKirKWCwWc/ToUdu8Pn36pFm/qSSZESNG2G63bt3auLu7m4MHD9rmnThxwvj6+pomTZrY5qVu++Hh4SYlJcU2v3///sbNzc1cuHDB4XMyJv2/uSTj7u5uDhw4YJu3fft2I8m8//77GS4vo/WU+tqYMWNGmrbMvr5u33ZT/9b58uUz586ds81ftGiRkWS+++67DPNWr17d7j0hPc2aNTNVq1a1y5GSkmIaNmxoypYta5vn6LUBAMCDjMM2AQDIgdq1a6erV69qyZIlunjxopYsWeJwxNWCBQvk7++vRx99VGfOnLFNtWrVko+Pj1atWmXrmydPHtv/L168qDNnzqhx48a6cuWK9uzZY7dcHx8fPfvss7bb7u7uqlu3rg4dOpSp5xAaGqpKlSrZbhtj9NVXX+mJJ56QMcYua0REhOLj47VlyxZJUkBAgP76669MHbJ2LwUEBEiSlixZouvXr2f5/oULF9ZTTz1lu+3n56cuXbpo69atio2NlXTz79W4cWMFBgbarYPw8HAlJydrzZo1dsts06aNChQokOnsixYtsjv8M7Nu3TYuX76sM2fOqGHDhjLGaOvWrVleXnJysn788Ue1bt1apUqVss0vVKiQOnXqpJ9//lkJCQl29+nVq5fd4ZeNGzdWcnKyjh49muXHl6Tw8HCVLl3adrtatWry8/PL9DbsiIeHh7p3755mflZeX+lp3769AgMDbbdTR73dKW9AQIB27typ/fv3p9t+7tw5rVy5Uu3atbPlOnPmjM6ePauIiAjt379fx48fv2M+AAAeVBTPAADIgQoUKKDw8HDNnTtXX3/9tZKTk/XMM8+k23f//v2Kj49XwYIFVaBAAbvp0qVLOnXqlK3vzp079dRTT8nf319+fn4qUKCArUAWHx9vt9yiRYumOY9UYGBgmvOoOVKyZEm726dPn9aFCxc0c+bMNDlTCxGpWQcNGiQfHx/VrVtXZcuWVZ8+fWyHH95PoaGhatOmjUaNGqX8+fPrySefVHR0tBITEzN1/zJlyqRZZ+XKlZMk29UL9+/fr2XLlqVZB+Hh4ZJk9/eS0q5HR9q3b6+HH35YL7zwgoKCgtShQwd9+eWXmS6kHTt
      "text/plain": [
       "<Figure size 1500x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABMYAAAGaCAYAAADkeU/nAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACK2UlEQVR4nOzdd3gUVf/+8Xs3pJBK7yX03qQjkCCRgKBG6SJNBUUQlS9IUzqCUkQFBBUQVKRZqA+KQBBpSpcmvQgk9IQaIDm/P/hlH5ZsQsIj2Q37fl3XXrAzZyf3bia7s58554zFGGMEAAAAAAAAuBmrswMAAAAAAAAAzkBhDAAAAAAAAG6JwhgAAAAAAADcEoUxAAAAAAAAuCUKYwAAAAAAAHBLFMYAAAAAAADgliiMAQAAAAAAwC1RGAMAAAAAAIBbojAGAAAAAAAAt0RhDAAAuJXg4GB16tTJ2TEeeRaLRUOGDEn3nztkyBBZLJaH/nMiIyNlsVgUGRn50H8WAAB4eCiMAQDggr766itZLBZZLBb9/vvvSdYbY1SwYEFZLBY1a9bMCQkfHXv27NGQIUN09OhRZ0dJV6dOndKQIUO0ffv2B97GsmXLnFL8wr/DXfd9AADuRmEMAAAX5uPjo9mzZydZvmbNGv3zzz/y9vZ2QqpHy549ezR06FC3Kw6cOnVKQ4cO/Z8LY0OHDnW47vr163r33XcfeNsP6t1339X169fT/edmRO667wMAcDcKYwAAuLCnnnpK8+fP1+3bt+2Wz549W1WrVlWePHmclOzBJSQk6MaNG86OkeHdvn1bN2/edHaMZPn4+ChTpkzp/nMzZcokHx+fdP+5AAAgY6IwBgCAC2vbtq3Onz+vFStW2JbdvHlTCxYs0AsvvODwMQkJCZowYYLKlSsnHx8f5c6dW6+++qouXrxo127hwoVq2rSp8uXLJ29vbxUrVkzDhw9XfHy8XbvQ0FCVL19ee/bsUYMGDeTr66v8+fPrww8/TNVzsFgs6tGjh7799luVK1dO3t7eWr58uSTp5MmTeumll5Q7d255e3urXLlymj59epJtfPrppypXrpx8fX2VNWtWVatWza4nXadOnRQcHJzkcfebb+qrr75Sy5YtJUkNGjSwDV9NnDdq8+bNCg8PV44cOZQ5c2YVKVJEL7300n2fc3BwsJo1a6ZffvlFlStXlo+Pj8qWLasffvghSdtLly7prbfeUsGCBeXt7a3ixYvrgw8+UEJCgq3N0aNHZbFYNHbsWE2YMEHFihWTt7e39uzZk2yGFStWqG7dusqSJYv8/f1VqlQpDRgwQNKd+bGqV68uSercubPteX/11VeSpLVr16ply5YqVKiQvL29VbBgQb399tt2PbE6deqkSZMmSZLt8Xe/1o7mGNu2bZuaNGmiwMBA+fv7q2HDhtq4caNdm8RhxOvWrVOvXr2UM2dO+fn56bnnntPZs2fv+9o7+p0n7oM//fSTypcvb9vXEvfD+/nnn38UEREhPz8/5cqVS2+//bbi4uIctt20aZMaN26soKAg+fr6KiQkROvWrbOtX7BggSwWi9asWZPksVOnTpXFYtGuXbuSzXLr1i0NHTpUJUqUkI+Pj7Jnz666devavUdI0r59+9SiRQtly5ZNPj4+qlatmhYtWmRbf799HwAAd5H+p/EAAECqBQcHq3bt2vruu+/UpEkTSdJ//vMfxcTEqE2bNvrkk0+SPObVV1/VV199pc6dO6tnz546cuSIJk6cqG3btmndunXy9PSUdOeLsb+/v3r16iV/f3+tWrVKgwYNUmxsrMaMGWO3zYsXL6px48Z6/vnn1apVKy1YsEB9+/ZVhQoVbLlSsmrVKs2bN089evRQjhw5FBwcrOjoaNWqVctWtMiZM6f+85//6OWXX1ZsbKzeeustSdIXX3yhnj17qkWLFnrzzTd148YN7dy5U5s2bUq2OJha9evXV8+ePfXJJ59owIABKlOmjCSpTJkyOnPmjBo1aqScOXOqX79+ypIli44ePeqwuOXIgQMH1Lp1a7322mvq2LGjZsyYoZYtW2r58uV68sknJUnXrl1TSEiITp48qVdffVWFChXS+vXr1b9/f50+fVoTJkyw2+aMGTN048YNde3aVd7e3sqWLZvDn7179241a9ZMFStW1LBhw+Tt7a2DBw/aCjRlypTRsGHDNGjQIHXt2lX16tWTJNWpU0eSNH/+fF27dk3dunVT9uzZ9ccff+jTTz/VP//8o/nz50u6s5+dOnVKK1as0Ndff33f12P37t2qV6+eAgMD9c4778jT01NTp05VaGio1qxZo5o1a9q1f+ONN5Q1a1YNHjxYR48e1YQJE9SjRw/NnTs3Va//vX7//Xf98MMPev311xUQEKBPPvlEzZs31/Hjx5U9e/ZkH3f9+nU1bNhQx48fV8+ePZUvXz59/fXXWrVqVZK2q1atUpMmTVS1alUNHjxYVqtVM2bM0BNPPKG1a9eqRo0aatq0qfz9/TVv3jyFhITYPX7u3LkqV66cypcvn2yeIUOGaNSoUXrllVdUo0YNxcbGavPmzdq6dattv9q9e7cef/xx5c+fX/369ZOfn5/mzZuniIgIff/993ruuedS3PcBAHArBgAAuJwZM2YYSebPP/80EydONAEBAebatWvGGGNatmxpGjRoYIwxpnDhwqZp06a2x61du9ZIMt9++63d9pYvX55keeL27vbqq68aX19fc+PGDduykJAQI8nMmjXLtiwuLs7kyZPHNG/e/L7PRZKxWq1m9+7ddstffvllkzdvXnPu3Dm75W3atDFBQUG2fM8++6wpV65cij+jY8eOpnDhwkmWDx482Nx7uFO4cGHTsWNH2/358+cbSWb16tV27X788Ufb7yCtChcubCSZ77//3rYsJibG5M2b11SpUsW2bPjw4cbPz8/s37/f7vH9+vUzHh4e5vjx48YYY44cOWIkmcDAQHPmzJn7/vyPPvrISDJnz55Nts2ff/5pJJkZM2YkWedo3xg1apSxWCzm2LFjtmXdu3dP8vomkmQGDx5sux8REWG8vLzMoUOHbMtOnTplAgICTP369W3LEvf9sLAwk5CQYFv+9ttvGw8PD3Pp0qVkn5Mxjn/nkoyXl5c5ePCgbdmOHTuMJPPpp5+muL0JEyYYSWbevHm2ZVevXjXFixe3228SEhJMiRIlTHh4uF3ua9eumSJFipgnn3zStqxt27YmV65c5vbt27Zlp0+fNlar1QwbNizFPJUqVbL7m3ekYcOGpkKFCnZ/xwkJCaZOnTqmRIkStmXJ7fsAALgThlICAODiWrVqpevXr2vJkiW6fPmylixZkmxPqfnz5ysoKEhPPvmkzp07Z7tVrVpV/v7+Wr16ta1t5syZbf+/fPmyzp07p3r16unatWvat2+f3Xb9/f314osv2u57eXmpRo0aOnz4cKqeQ0hIiMqWLWu7b4zR999/r6efflrGGLus4eHhiomJ0datWyVJWbJk0T///KM///wzVT/r35IlSxZJ0pIlS3Tr1q00Pz5fvnx67rnnbPcDAwPVoUMHbdu2TVFRUZLu/L7q1aunrFmz2r0GYWFhio+P12+//Wa3zebNmytnzpypzr5w4UK7IZmpdfe+cfXqVZ07d0516tSRMUbbtm1L8/bi4+P1yy+/KCIiQkWLFrUtz5s3r1544QX9/vvvio2NtXtM165d7YZE1qtXT/Hx8Tp27Fiaf74khYWFqVixYrb7FStWVGBg4H334WXLlilv3rxq0aKFbZmvr6+6du1q12779u06cOCAXnjhBZ0/f972u7x69aoaNmyo3377zfa7aN26tc6cOWM3bHHBggVKSEhQ69atU8yTJUsW7d69WwcOHHC4/sKFC1q1apVatWpl+7s+d+6czp8/r/DwcB04cEAnT55M8WcAAOBOKIwBAODicubMqbCwMM2ePVs//PCD4uPj7b6k3+3AgQOKiYlRrly5lDNnTrvblStXdObMGVvb3bt367nnnlNQUJACAwOVM2dOW/ErJibGbrsFChRIMm9T1qxZk8xblpwiRYrY3T979qwuXbqkzz//PEnOzp0
      "text/plain": [
       "<Figure size 1500x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABMYAAAGaCAYAAADkeU/nAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACHy0lEQVR4nOzdd3gUVf/+8Xs3IQmkUaVD6L0oHYVQIkVAo1JFqoIiYOGho4QigiKICogFEFSkqfQHRZp0pXfpRSB0EmqA5Pz+4Jf9smQTEh7JTtj367r2gp2Z3b13Mjs7+5lzztiMMUYAAAAAAACAh7G7OwAAAAAAAADgDhTGAAAAAAAA4JEojAEAAAAAAMAjURgDAAAAAACAR6IwBgAAAAAAAI9EYQwAAAAAAAAeicIYAAAAAAAAPBKFMQAAAAAAAHgkCmMAAAAAAADwSBTGAACARwkJCVH79u3dHeORZ7PZNGjQoFR/3UGDBslms6X66wIAgLSJwhgAABb07bffymazyWazafXq1QnmG2OUN29e2Ww2NW7c2A0JHx27d+/WoEGDdOTIEXdHSVUnT57UoEGDtHXr1gd+jkWLFrml+JWa/o31lBzTpk3TmDFjHuprJOaDDz7QnDlz3PLaAAC4G4UxAAAszM/PT9OmTUswfeXKlfrnn3/k6+vrhlSPlt27d2vw4MEeWRgbPHjw/1wYGzx4sMt5169f17vvvvvAz/2g3n33XV2/fv1fe75/Yz0lB4UxAADcg8IYAAAW9swzz2jWrFm6ffu20/Rp06apQoUKypEjh5uSPbi4uDjduHHD3THSvNu3b+vmzZvujpEoPz8/eXt7p/rrent7y8/PL9VfFwAApE0UxgAAsLBWrVrp/PnzWrJkiWPazZs3NXv2bL300ksuHxMXF6cxY8aoVKlS8vPzU/bs2fXaa6/p4sWLTsvNnTtXjRo1Uq5cueTr66tChQpp6NChio2NdVquVq1aKl26tHbv3q3atWsrQ4YMyp07tz766KNkvQebzaZu3brphx9+UKlSpeTr66vFixdLkk6cOKGOHTsqe/bs8vX1ValSpTRp0qQEz/H555+rVKlSypAhgzJlyqSKFSs6taRr3769QkJCEjzufuNNffvtt2rWrJkkqXbt2o7uqytWrJAkbdy4UfXr11fWrFmVPn16FShQQB07drzvew4JCVHjxo3122+/qXz58vLz81PJkiX1888/J1j20qVLevvtt5U3b175+vqqcOHC+vDDDxUXF+dY5siRI7LZbPr44481ZswYFSpUSL6+vtq9e3eiGZYsWaKnnnpKGTNmVEBAgIoVK6b+/ftLklasWKFKlSpJkjp06OB4399++60kadWqVWrWrJny5csnX19f5c2bV++8845TS6z27dtr3LhxkuR4/N3r2tUYY1u2bFHDhg0VFBSkgIAA1a1bV+vXr3daJr4b8Zo1a9SjRw9ly5ZN/v7+ev7553X27Nn7rntXf/P4bXDOnDkqXbq0Y1uL3w4Tc7/1JEkbNmxQgwYNFBwcrAwZMig0NFRr1qxxep7Lly/r7bffVkhIiHx9ffXYY4/p6aef1ubNmyXd+YwtXLhQR48edbyGq+35bkn9fePFxMQoIiJChQsXdvwde/furZiYGKd1c/XqVU2ZMsXx2ozBBwDwJKl/Gg8AACRbSEiIqlWrph9//FENGzaUJP33v/9VVFSUWrZsqc8++yzBY1577TV9++236tChg958800dPnxYY8eO1ZYtW7RmzRqlS5dO0p0CREBAgHr06KGAgAAtW7ZMAwcOVHR0tEaOHOn0nBcvXlSDBg30wgsvqHnz5po9e7b69OmjMmXKOHIlZdmyZZo5c6a6deumrFmzKiQkRKdPn1bVqlUdRYts2bLpv//9r1555RVFR0fr7bffliR9/fXXevPNN9W0aVO99dZbunHjhrZv364NGzYkWhxMrpo1a+rNN9/UZ599pv79+6tEiRKSpBIlSujMmTOqV6+esmXLpr59+ypjxow6cuSIy+KWK/v371eLFi30+uuvq127dpo8ebKaNWumxYsX6+mnn5YkXbt2TaGhoTpx4oRee+015cuXT2vXrlW/fv106tSpBF3rJk+erBs3bqhz587y9fVV5syZXb72rl271LhxY5UtW1ZDhgyRr6+vDhw44CjYlChRQkOGDNHAgQPVuXNn1ahRQ5JUvXp1SdKsWbN07do1denSRVmyZNGff/6pzz//XP/8849mzZol6c52dvLkSS1ZskTffffdfdfHrl27VKNGDQUFBal3795Kly6dvvzyS9WqVUsrV65UlSpVnJbv3r27MmXKpIiICB05ckRjxoxRt27dNGPGjGSt/3utXr1aP//8s9544w0FBgbqs88+04svvqhjx44pS5YsLh9zv/W0bNkyNWzYUBUqVFBERITsdrsmT56sOnXqaNWqVapcubIk6fXXX9fs2bPVrVs3lSxZUufPn9fq1au1Z88ePfHEExowYICioqL0zz//6JNPPpEkBQQEJLkuk/r7SncK5M8++6xWr16tzp07q0SJEtqxY4c++eQT7du3z9F18rvvvtOrr76qypUrq3PnzpKkQoUKPdA6BgAgTTIAAMByJk+ebCSZv/76y4wdO9YEBgaaa9euGWOMadasmaldu7Yxxpj8+fObRo0aOR63atUqI8n88MMPTs+3ePHiBNPjn+9ur732msmQIYO5ceOGY1poaKiRZKZOneqYFhMTY3LkyGFefPHF+74XScZut5tdu3Y5TX/llVdMzpw5zblz55ymt2zZ0gQHBzvyPffcc6ZUqVJJvka7du1M/vz5E0yPiIgw9x7u5M+f37Rr185xf9asWUaSWb58udNyv/zyi+NvkFL58+c3ksxPP/3kmBYVFWVy5sxpHn/8cce0oUOHGn9/f7Nv3z6nx/ft29d4eXmZY8eOGWOMOXz4sJFkgoKCzJkzZ+77+p988omRZM6ePZvoMn/99ZeRZCZPnpxgnqttY/jw4cZms5mjR486pnXt2jXB+o0nyURERDjuh4eHGx8fH3Pw4EHHtJMnT5rAwEBTs2ZNx7T4bT8sLMzExcU5pr/zzjvGy8vLXLp0KdH3ZIzrv7kk4+PjYw4cOOCYtm3bNiPJfP7550k+X2LrKS4uzhQpUsTUr1/fKee1a9dMgQIFzNNPP+2YFhwcbLp27Zrk6zRq1MjlNuxKcv6+3333nbHb7WbVqlVO0ydMmGAkmTVr1jim+fv7O30mAADwJHSlBADA4po3b67r169rwYIFunz5shYsWJBoS6lZs2YpODhYTz/9tM6dO+e4VahQQQEBAVq+fLlj2fTp0zv+f/nyZZ07d041atTQtWvXtHfvXqfnDQgI0Msvv+y47+Pjo8qVK+vQoUPJeg+hoaEqWbKk474xRj/99JOaNGkiY4xT1vr16ysqKsrRzSxjxoz6559/9NdffyXrtf4tGTNmlCQtWLBAt27dSvHjc+XKpeeff95xPygoSG3bttWWLVsUGRkp6c7fq0aNGsqUKZPTOggLC1NsbKz++OMPp+d88cUXlS1btmRnnzt3rlOXzOS6e9u4evWqzp07p+rVq8sYoy1btqT4+WJjY/Xbb78pPDxcBQsWdEzPmTOnXnrpJa1evVrR0dFOj+ncubNTl8gaNWooNjZWR48eTfHrS1JYWJhTS6iyZcsqKCgo2dvwvbZu3ar9+/frpZde0vnz5x1/u6tXr6pu3br6448/HOs+Y8aM2rBhg06ePPlAr3Wv5Px9Z82apRIlSqh48eJO21adOnUkyWlfAACAJ6MwBgCAxWXLlk1hYWGaNm2afv75Z8XGxqpp06Yul92/f7+ioqL02GOPKVu2bE63K1eu6MyZM45ld+3apeeff17BwcEKCgpStmzZHMWvqKgop+fNkydPgnGbMmXKlGDcssQUKFDA6f7Zs2d16dIlffXVVwlydujQQZIcWfv06aOAgABVrlxZRYoUUdeuXROM4fQwhIaG6sUXX9TgwYOVNWtWPffcc5o8ebLT+ExJKVy4cIJ1VrRoUUlyXAFz//79Wrx4cYJ1EBYWJklOfy8p4XpMTIs
      "text/plain": [
       "<Figure size 1500x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i, split in enumerate(splits): \n",
    "    results_df = pd.DataFrame(results[i]).transpose()\n",
    "\n",
    "    # for station, values in results_df[['mae']].itertuples():\n",
    "    #     results_df.loc[station, 'mae'] =  np.mean([v for v in values if v is not None]) \n",
    "    # for station, values in results_df[['loss']].itertuples():\n",
    "    #     results_df.loc[station, 'loss'] =  np.mean([v for v in values if v is not None]) \n",
    "        \n",
    "    results_df.plot(kind='bar', figsize=(15,4), title=f\"Mean results per station in {split} set\")\n",
    "\n",
    "    results[i] = results_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f02c9173-986c-45d5-973c-a68406ff20df",
   "metadata": {},
   "source": [
    "### Check predictions for stations with highest MAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "66a99163-04cd-4ecb-a3c8-fa8dc112c435",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "highest mean MAE in dev set: 0.13 for station: LUBW\n",
      "highest mean MAE in test set: 0.10 for station: RYNR\n"
     ]
    }
   ],
   "source": [
    "dev_res = results[1]\n",
    "station_with_worst_res_dev_set = dev_res[dev_res.mae == dev_res.mae.max()].index[0]\n",
    "highest_dev_mae = dev_res.loc[station_with_worst_res_dev_set, 'mae']\n",
    "\n",
    "test_res = results[2]\n",
    "station_with_worst_res_test_set = test_res[test_res.mae == test_res.mae.max()].index[0]\n",
    "highest_test_mae = test_res.loc[station_with_worst_res_test_set, 'mae']\n",
    "print(f\"highest mean MAE in dev set: {highest_dev_mae:.2f} for station: {station_with_worst_res_dev_set}\")\n",
    "print(f\"highest mean MAE in test set: {highest_test_mae:.2f} for station: {station_with_worst_res_test_set}\")\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "73a07433-1663-440c-bce3-d73988e9685b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_sample(sample, model, i, desc=None): \n",
    "    fig = plt.figure(figsize=(15, 10))\n",
    "    \n",
    "    axs = fig.subplots(2, 1, sharex=True, gridspec_kw={\"hspace\": 0, \"height_ratios\": [3,  2]})\n",
    "    axs[0].plot(sample[\"X\"][0].T, label='x')\n",
    "    plt.legend()\n",
    "    axs[1].plot(sample[\"y\"][0].T, label='y')\n",
    "    \n",
    "    model.eval()  # close the model for evaluation\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        pred = model(torch.tensor(sample[\"X\"], device=model.device).unsqueeze(0))  # Add a fake batch dimension\n",
    "        pred = pred[0].cpu().numpy()\n",
    "        \n",
    "        axs[1].plot(pred[0], label='pred', color='orange')\n",
    "        plt.legend()\n",
    "\n",
    "        pred_pick_idx  = np.argmax(pred[0])\n",
    "        true_pick_idx = np.argmax(sample['y'][0])\n",
    "\n",
    "        \n",
    "        \n",
    "        mae_error = np.abs(pred_pick_idx - true_pick_idx) /100 #mae in seconds\n",
    "\n",
    "        fig.suptitle(f\"Predictions for sample: {i} {desc}, mae: {mae_error}s\")\n",
    "        \n",
    "        plt.show()\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d28d456a-4034-4eea-ac28-ed29ddb98d75",
   "metadata": {},
   "source": [
    "### Find random samples that reproduce obtained results \n",
    "\n",
    "the model makes at least as bad predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "235ae13e-3250-49e6-8bb2-765c037b0870",
   "metadata": {},
   "outputs": [],
   "source": [
    "##### dev set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5e3829f5-360a-431f-9fdd-bdeff6a5a1e4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev (2773, 17) 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.12958333333333336 LUBW\n",
      "[1.06 0.4  0.33 0.28 0.23 0.1  0.08 0.07 0.07 0.07 0.07 0.06 0.05 0.05\n",
      " 0.05 0.04 0.03 0.03 0.01 0.01 0.01 0.01 0.   0.  ]\n",
      "2 1.06\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNIAAAORCAYAAAA3ZI+fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd7wU1dkH8N/urXQLIBCIosaOkpDE3lHsmmZJEXs3RmJM9I1ii9hiiRqxIrao2CtVUEGK0gTpvV8u5fZ7t8yc949ld885M+fs7N69Bfh9349vuLuzM7Ozs7MzzzzPc0JCCAEiIiIiIiIiIiKyCrf0ChAREREREREREW0PGEgjIiIiIiIiIiIKgIE0IiIiIiIiIiKiABhIIyIiIiIiIiIiCoCBNCIiIiIiIiIiogAYSCMiIiIiIiIiIgqAgTQiIiIiIiIiIqIAGEgjIiIiIiIiIiIKgIE0IiIiIiIiIiKiABhIIyIiAMBee+2FSy65JPX3hAkTEAqFMGHChLwtIxQK4a677srb/PLt4Ycfxt57742CggL07du3pVenVWrtn6FNTU0NrrjiCnTr1g2hUAh/+ctfWnqVWq0VK1YgFArh5ZdfbulV2aFdcskl2GuvvVp6NYiIiCgLDKQREbUCL7/8MkKhUOq/0tJS7LfffrjhhhtQVlbW0quXlc8++2y7DLSMHj0at956K44++mgMGzYM999/f0uv0k5r9erVuPvuu/HLX/4Su+66Kzp37owTTjgBY8eObdR877//frz88su49tpr8eqrr+JPf/pTnta4ZcybNw933XUXVqxYkfM83njjDTz++ON5W6d8uOSSS9C+fXvj88nj5Xfffef7/FlnneUJTsnH11AohHbt2uGggw7Cfffdh7q6OgDAtGnTEAqF8Nhjj3nmee655yIUCmHYsGGe54477jj86Ec/yuId7lha4z7U3EaPHo3LL78chxxyCAoKCnIKjn700Uf42c9+htLSUvz4xz/G4MGDEY/HfacdO3YsTjrpJHTq1AkdOnRAv3798NZbbzXyXRARUVCFLb0CRESUds8996B3795oaGjAxIkT8cwzz+Czzz7D3Llz0bZt22Zdl+OOOw719fUoLi7O6nWfffYZnn76ad9gWn19PQoLW+dPzxdffIFwOIwXX3wx6/dM+fXhhx/iwQcfxHnnnYeBAwciHo/jlVdewSmnnIKXXnoJl156aU7z/eKLL3DEEUdg8ODBeV7jljFv3jzcfffdOOGEE3LOanrjjTcwd+5cT3bennvuifr6ehQVFTV+RVuJU045BRdffDGARHbi119/jTvuuAOzZ8/GiBEj8LOf/Qxt27bFxIkTcfPNNyuv/eabb1BYWIhJkyYp+180GsW3336Ls88+u1nfS2ti2od2Jm+88Qbeeust/OxnP0OPHj2yfv3nn3+O8847DyeccAKefPJJzJkzB/fddx82btyIZ555Rpl22LBhuPzyy3HKKafg/vvvR0FBARYuXIjVq1fn6+0QEVEGrfNqhohoJ3X66afj5z//OQDgiiuuwO67745HH30UH374IS666CLf19TW1qJdu3Z5X5dwOIzS0tK8zjPf88unjRs3ok2bNnkLogkh0NDQgDZt2uRlfjuTE088EatWrULnzp1Tj11zzTXo27cv7rzzzpwDaRs3bsRBBx2UcbqGhgYUFxcjHN55E/eTmbE7kv322w9//OMfU39fc801iEajeO+999DQ0IDS0lIcfvjhmDRpkvK6hQsXYtOmTfj973+PiRMnKs9Nnz4dDQ0NOOaYY5rlPVDrdP/99+P5559HUVERzjrrLMydOzer199yyy049NBDMXr06NTNpo4dO+L+++/HTTfdhAMOOABAouT6+uuvx4033ognnngi7++DiIiC2XnPEImItgMnnXQSAGD58uUA0iVPS5cuxRlnnIEOHTrgD3/4AwDAdV08/vjjOPjgg1FaWoo99tgDV199NbZu3arMUwiB++67Dz179kTbtm1x4okn4ocffvAs29QjberUqTjjjDOw6667ol27djj00ENTJ/SXXHIJnn76aQBqKVWSX3+tmTNn4vTTT0fHjh3Rvn17nHzyyZgyZYoyTbKUa9KkSRg0aBC6dOmCdu3a4Ve/+hXKy8uVab/77jsMGDAAnTt3Rps2bdC7d29cdtll1u2cLNmqra1NrXOyN1Q8Hse9996LffbZByUlJdhrr71w++23IxKJKPPYa6+9cNZZZ2HUqFH4+c9/jjZt2uDZZ581LnPx4sX4zW9+g27duqG0tBQ9e/bEhRdeiMrKytQ0w4YNw0knnYSuXbuipKQEBx10kCc7QV72hAkTUsvu06dP6rN777330KdPH5SWlqJfv36YOXOm8vrkfrVs2TIMGDAA7dq1Q48ePXDPPfdACGHddgCwdu1aXHbZZdhjjz1QUlKCgw8+GC+99JJnulWrVmHBggUZ53fwwQcrQTQAKCkpwRlnnIE1a9aguro64zxkyX15+fLl+PTTT1Of8YoVK1LPvfnmm/jnP/+JH/3oR2jbti2qqqoAACNGjEC/fv3Qpk0bdO7cGX/84x+xdu1aZf7J7bdq1SqcddZZaN++PX70ox+lvgtz5szBSSedhHbt2mHPPffEG2+8EWi933zzTfTr1w8dOnRAx44d0adPn9R37eWXX8bvfvc7AInAY/I9JT/zDz/8EGeeeSZ69OiBkpIS7LPPPrj33nvhOE5q/ieccAI+/fRTrFy5MvX6ZGabqUfaF198gWOPPRbt2rXDLrvsgnPPPRfz589XprnrrrsQCoWwZMkSXHLJJdhll13QqVMnXHrppakyytYi2S8vGbw45phjUFZWhiVLlqSmmTRpEjp27IirrroqFVSTn0u+LpMPPvgAhxxyCEpLS3HIIYfg/fff950uyLH8rLPOwt577+37+iOPPDJ1Q8YkyPEHAF577bXU/r/bbrvhwgsvVDKfbPtQNhr7HdqyZQtuueUW9OnTB+3bt0fHjh1x+umnY/bs2Z5lRSIRDB48GPvuuy9KSkrQq1cv3HrrrZ5j+qZNm7BgwYJA+2yPHj1yzt6cN28e5s2bh6uuukrJ2L7uuusghMA777yTemzo0KFwHAf33HMPgERmpekYbTt+EBFR4zAjjYioFVu6dCkAYPfdd089Fo/HMWDAABxzzDF45JFHUiWfV199NV5++WVceuml+POf/4zly5fjqaeewsyZMzFp0qTUSf6dd96J++67D2eccQbOOOMMzJgxA6eeeiqi0WjG9RkzZgzOOussdO/eHTfddBO6deuG+fPn45NPPsFNN92Eq6++GuvWrcOYMWPw6quvZpzfDz/8gGOPPRYdO3bErbfeiqKiIjz77LM44YQT8OWXX+Lwww9Xpr/xxhux6667YvDgwVixYgUef/xx3HDDDaneMBs3bsSpp56KLl264B//+Ad22WUXrFixAu+99551PV599VU899xzmDZtGl544QUAwFFHHQUgkRk4fPhw/Pa3v8Vf//pXTJ06FUOGDMH8+fM9F8ILFy7ERRddhKuvvhpXXnkl9t9/f9/lRaNRDBgwAJFIBDfeeCO6deuGtWvX4pNPPkFFRQU6deoEAHjmmWdw8MEH45xzzkFhYSE+/vhjXHfddXBdF9dff70yzyVLluD3v/89rr76avzxj3/EI488grPPPhtDhw7F7bffjuuuuw4AMGTIEJx//vlYuHChknHlOA5OO+00HHHEEXjooYcwcuTIVI+e5EWbn7KyMhxxxBEIhUK44YYb0KVLF3z++ee4/PLLUVVVpZR7XXzxxfjyyy8DBef8bNiwAW3bts26zPnAAw/Eq6++iptvvhk9e/bEX//6VwBAly5dUv3F7r33XhQXF+OWW25BJBJBcXFx6vv0i1/8AkOGDEFZWRmeeOIJTJo0CTNnzsQuu+ySWobjODj99NNx3HHH4aGHHsLrr7+OG264Ae3atcP//d//4Q9/+AN+/etfY+jQobj44otx5JFHonfv3sZ1HjNmDC666CKcfPLJePDBBwEA8+fPx6RJk3DTTTfhuOOOw5///Gf85z//we23344DDzw
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 0.4\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNIAAAORCAYAAAA3ZI+fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd5wU5f0H8M/uNToWQFAJIsYWCwnGrlhQxBLNL7ElBsQeWyIxJpoo1hC7xl6xK2JNLBQRVJp0pEvvHHfA3XFt93bn+f1x7O7zPDPPM7PL7hXu8369eLG3OzszOzszO893vs/3CQkhBIiIiIiIiIiIiMgq3NgrQERERERERERE1BwwkEZERERERERERBQAA2lEREREREREREQBMJBGREREREREREQUAANpREREREREREREATCQRkREREREREREFAADaURERERERERERAEwkEZERERERERERBQAA2lEREREREREREQBMJBGRERG++23Hy6//PLk3xMmTEAoFMKECROytoxQKIS77747a/PLtocffhj7778/8vLy0Lt378ZenSapqX+HNpWVlbjqqqvQtWtXhEIh/PnPf27sVWqyVq1ahVAohNdee62xV2WXdvnll2O//fZr7NUgIiIiAwbSiIiaqNdeew2hUCj5r1WrVjjwwANx4403ori4uLFXLy1ffPFFswy0jBkzBrfddhtOOOEEDB8+HP/6178ae5VarJqaGlx55ZU47LDD0LFjR7Rr1w5HHnkknnzySdTV1WU833/961947bXX8Mc//hFvvvkm/vCHP2RxrRvewoULcffdd2PVqlUZz+Odd97BE088kbV1yobLL78c7dq1M76eOF/OmDHD8/Vzzz3XFZySz6+hUAht27bFoYceivvvvx/V1dUAgGnTpiEUCuHxxx93zfP8889HKBTC8OHDXa+dfPLJ2GeffdL4hLuWprgPNYbJkyfjxBNPRJs2bdC1a1fcfPPNqKysTHs+EydOTO6npaWlOVhTIiJKR35jrwAREdnde++96NmzJ2prazFx4kQ899xz+OKLLzB//ny0adOmQdfl5JNPRk1NDQoLC9N63xdffIFnnnnGM5hWU1OD/Pym+XP09ddfIxwO45VXXkn7M1N21dTUYMGCBTj77LOx3377IRwOY/Lkybjlllvw/fff45133slovl9//TWOPfZYDB06NMtr3DgWLlyIe+65B6ecckrGWU3vvPMO5s+f78rO69GjB2pqalBQULDzK9pEnHHGGRg4cCCA+uzE7777DnfeeSfmzp2LkSNH4he/+AXatGmDiRMn4pZbblHeO3nyZOTn52PSpEkYPHhw8vloNIrp06fjvPPOa9DP0pSY9qGWZM6cOTj99NNxyCGH4LHHHsO6devwyCOPYOnSpfjyyy8Dz8dxHNx0001o27YtqqqqcrjGREQUVNNsuRARUdKAAQNw1FFHAQCuuuoq7Lnnnnjsscfw6aef4tJLL/V8T1VVFdq2bZv1dQmHw2jVqlVW55nt+WXT5s2b0bp166wF0YQQqK2tRevWrbMyv5Zkjz32wNSpU5XnrrvuOnTs2BFPP/00HnvsMXTt2jXt+W7evBmHHnqo73S1tbUoLCxEONxyk/kTmbG7kgMPPBCXXXZZ8u/rrrsO0WgUH330EWpra9GqVSscc8wxmDRpkvK+JUuWoLS0FL/73e8wceJE5bWZM2eitrYWJ554YoN8Bmqa7rjjDuy+++6YMGECOnToAKC+XMLVV1+NMWPG4Mwzzww0nxdffBFr167FVVddhSeffDKXq0xERAG13KtBIqJm6rTTTgMArFy5EkCqy9Py5ctx9tlno3379vj9738PoP5O9hNPPIGf/exnaNWqFfbaay9ce+212LZtmzJPIQTuv/9+7LvvvmjTpg1OPfVULFiwwLVsU42077//HmeffTZ23313tG3bFkcccUTygv/yyy/HM888A0DtSpXgVV9r9uzZGDBgADp06IB27drh9NNPdwVREl25Jk2ahCFDhqBz585o27Ytfv3rX6OkpESZdsaMGejfvz86deqE1q1bo2fPnrjiiius2znRZauqqiq5zonaULFYDPfddx969eqFoqIi7LfffrjjjjsQiUSUeey3334499xzMXr0aBx11FFo3bo1XnjhBeMyly5dit/85jfo2rUrWrVqhX333ReXXHIJysvLk9MMHz4cp512Grp06YKioiIceuiheO6551zzSix7woQJyWUffvjhye/uo48+wuGHH45WrVqhT58+mD17tvL+xH61YsUK9O/fH23btsXee++Ne++9F0II67YDgPXr1+OKK67AXnvthaKiIvzsZz/Dq6++6ppuzZo1WLx4se/8TBJZV2VlZWm9L7Evr1y5Ep9//nnyO161alXytffeew///Oc/sc8++6BNmzaoqKgAAIwcORJ9+vRB69at0alTJ1x22WVYv369Mv/E9luzZg3OPfdctGvXDvvss0/yWJg3bx5OO+00tG3bFj169AicUffee++hT58+aN++PTp06IDDDz88eay99tpruPDCCwEAp556avIzJb7zTz/9FOeccw723ntvFBUVoVevXrjvvvsQj8eT8z/llFPw+eefY/Xq1cn3J7axqUba119/jZNOOglt27bFbrvthvPPPx+LFi1Sprn77rsRCoWwbNkyXH755dhtt93QsWNHDB48ONmNsqlI1MtLZMqeeOKJKC4uxrJly5LTTJo0CR06dMA111yTDKrJryXe5+eTTz7BYYcdhlatWuGwww7Dxx9/7DldkHP5ueeei/3339/z/ccdd1zyhoxJkPMPALz11lvJ/X+PPfbAJZdcgrVr1yZft+1D6djZY2jr1q249dZbcfjhh6Ndu3bo0KEDBgwYgLlz57qWFYlEMHToUBxwwAEoKipC9+7dcdttt7nO6aWlpVi8eLHvPltRUYGxY8fisssuSwbRAGDgwIFo164d3n///UDbYOvWrfjnP/+Je++9F7vttpvnNEG/NyIiyh5mpBERNTPLly8HAOy5557J52KxGPr3748TTzwRjzzySLLL57XXXovXXnsNgwcPxs0334yVK1fi6aefxuzZszFp0qRkF6277roL999/P84++2ycffbZmDVrFs4880xEo1Hf9Rk7dizOPfdcdOvWDX/605/QtWtXLFq0CJ999hn+9Kc/4dprr8WGDRswduxYvPnmm77zW7BgAU466SR06NABt912GwoKCvDCCy/glFNOwTfffINjjjlGmf6mm27C7rvvjqFDh2LVqlV44okncOONN2LEiBEA6jOOzjzzTHTu3Bl///vfsdtuu2HVqlX46KOPrOvx5ptv4sUXX8S0adPw8ssvAwCOP/54APWZga+//jp++9vf4i9/+Qu+//57DBs2DIsWLXI1hJcsWYJLL70U1157La6++mocdNBBnsuLRqPo378/IpEIbrrpJnTt2hXr16/HZ599hrKyMnTs2BEA8Nxzz+FnP/sZfvWrXyE/Px//+9//cP3118NxHNxwww3KPJctW4bf/e53uPbaa3HZZZfhkUcewXnnnYfnn38ed9xxB66//noAwLBhw3DRRRdhyZIlSsZVPB7HWWedhWOPPRYPPfQQRo0ahaFDhyIWi+Hee+81brvi4mIce+yxCIVCuPHGG9G5c2d8+eWXuPLKK1FRUaF09xo4cCC++eabQMG5xHaqqKhATU0NZsyYgUceeQQ9evTAAQccEOj9CYcccgjefPNN3HLLLdh3333xl7/8BQDQuXPnZH2x++67D4WFhbj11lsRiURQWFiYPJ5++ctfYtiwYSguLsaTTz6JSZMmYfbs2UpjNx6PY8CAATj55JPx0EMP4e2338aNN96Itm3b4h//+Ad+//vf4//+7//w/PPPY+DAgTjuuOPQs2dP4zqPHTsWl156KU4//XQ8+OCDAIBFixZh0qRJ+NOf/oSTTz4ZN998M/7zn//gjjvuwCGHHJL8rEB9oK1du3YYMmQI2rVrh6+//hp33XUXKioq8PDDDwMA/vGPf6C8vBzr1q1L1gWz1Sb76quvMGDAAOy///64++6
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13 0.33\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNIAAAORCAYAAAA3ZI+fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd5wU9f0/8NfuXuVoIlVFwB4MSoKxYEEjil2S2JOvWGM0Ro2J/tQk9ogt9oK9V+yVKqh0EJDeezs4jutly3x+f+zN7Ofzmc/Mzl7hOO71zCMPub3ZmdnZ2b2d177fn09ICCFAREREREREREREvsLNvQNEREREREREREQtAYM0IiIiIiIiIiKiABikERERERERERERBcAgjYiIiIiIiIiIKAAGaURERERERERERAEwSCMiIiIiIiIiIgqAQRoREREREREREVEADNKIiIiIiIiIiIgCYJBGREREREREREQUAIM0IqJWpnfv3rjsssucnydOnIhQKISJEyc22jZCoRDuvvvuRltfY3vkkUew3377IRKJoH///s29O7ukXf05bCwzZ87EwIEDUVBQgFAohLlz5zb3Lu2y7r77boRCoebejd1ea3ntERERtVQM0oiIdqLXX38doVDI+X9eXh4OOuggXH/99SgsLGzu3cvIN9980yIv9saMGYNbb70Vxx57LF577TU88MADzb1Lrdrzzz+P888/H/vuuy9CoZAS8sp++OEHnHPOOejZsyfy8vLQvXt3nHbaaZg8eXK9tx2LxXD++eejuLgYjz/+ON566y306tWr3uvbFbz77rt44okn6n3/qqoq3H333Y0arDeGUCiE66+/3vP3J554In75y18af1dUVOQKp/T34lAohK5du+Kkk07Ct99+6yx33XXXIRwOo7i4WFlncXExwuEwcnNzUVNTo/xu1apVCIVCuOOOO+rxSFu+XfUc2tksy8LDDz+MPn36IC8vD4cddhjee++9QPfN5P3ugQcewNFHH40uXbogLy8PBx54IG666SZs27atsR8SERHVyWruHSAiao3uvfde9OnTBzU1NZg0aRKef/55fPPNN1iwYAHatGmzU/flhBNOQHV1NXJycjK63zfffINnn33WGKZVV1cjK2vX/BPz3XffIRwO45VXXsn4MVPje+ihh1BeXo4jjzwSmzdv9lxu2bJlCIfD+Mtf/oLu3btjx44dePvtt3HCCSfg66+/xmmnnZbxtleuXIm1a9fipZdewlVXXdWQh7HLePfdd7FgwQLcdNNN9bp/VVUV7rnnHgDJcEr273//G7fddlsD93DXYr8XCyFQWFiI119/HWeccQa+/PJLnHXWWTjuuOPw/PPPY/LkyTj77LOd+02ZMgXhcBixWAyzZs3Ccccd5/zODjvk21oTv3OoNfnXv/6FBx98EFdffTV+85vf4PPPP8cll1yCUCiEiy66yPe+mbzf/fTTT+jfvz8uuugitGvXDosXL8ZLL72Er7/+GnPnzkVBQUFTP1QiolZn17zKISLazZ1++uk44ogjAABXXXUV9txzTzz22GP4/PPPcfHFFxvvU1lZ2SQfiMPhMPLy8hp1nY29vsa0detW5OfnN1qIJoRATU0N8vPzG2V9rc3333/vVKO1bdvWc7mrrrrKFXZdd9112G+//fDEE0/UK0jbunUrAKBjx45pl22q119LkpWVtcsG5PUlvxcDwJVXXolu3brhvffec4I0AJg0aZISpE2ePBmHHXYYqqurMWnSJCU0mzRpEsLhMAYOHLjzHgjtUjZu3Ij//e9/+Otf/4pnnnkGQPI9bNCgQbjllltw/vnnIxKJeN4/k/e7jz/+2HX/Y445Bueddx6+/PLLtKEdERFljq2dRES7gN/+9rcAgNWrVwMALrvsMrRt2xYrV67EGWecgXbt2uGPf/wjgGS7yBNPPIFDDz0UeXl56NatG6655hrs2LFDWacQAvfffz/22WcftGnTBieddBIWLlzo2rbXGGnTp0/HGWecgT322AMFBQU47LDD8OSTTzr79+yzzwKA0hplM43xM2fOHJx++ulo37492rZti5NPPhnTpk1TlrHbrSZPnoybb74ZXbp0QUFBAX73u9+52lRmzZqFIUOGoHPnzsjPz0efPn1wxRVX+B7nUCiE1157DZWVlc4+v/766wCAeDyO++67D/vvvz9yc3PRu3dv3HHHHaitrVXW0bt3b5x11lkYPXo0jjjiCOTn5+OFF17w3Oby5cvxhz/8Ad27d0deXh722WcfXHTRRSgtLXWWee211/Db3/4WXbt2RW5uLvr27Yvnn3/etS572xMnTnS23a9fP+e5++STT9CvXz/k5eVhwIABmDNnjnJ/+7xatWoVhgwZgoKCAuy111649957IYTwPXZA8uLwiiuuQLdu3ZCbm4tDDz0Ur776qmu5devWYcmSJWnXBwC9evWq97hbbdq0QZcuXVBSUpLxfS+77DIMGjQIAHD++ecjFAo51TN+r7/Kykr84x//QM+ePZGbm4uDDz4Yjz76qOv42a2II0eORN++fZGfn49jjjkG8+fPBwC88MILOOCAA5CXl4cTTzwRa9asSbvP5eXluOmmm9C7d2/k5uaia9euOOWUUzB79mwAyeqfr7/+GmvXrnXO7969ewMAotEo7rzzTgwYMAAdOnRAQUEBjj/+eEyYMMFZ/5o1a9ClSxcAwD333OOsw34tm8ZIy/R1M2nSJBx55JHIy8vDfvvthzfffDPt496ZOnbsiPz8fCcw3HfffdGzZ09XS93kyZNx7LHHYuDAgcbfHXrooWkD2traWvz9739Hly5d0K5dO5xzzjnYsGGDcdl0r73CwkJkZWU5lWCypUuXIhQKOYGOl/fffx8DBgxAu3bt0L59e/Tr1895z7eVlJTgpptucs7/Aw44AA899BAsywKQ/hzKRENfQz/++KPTNp6bm4uePXvi73//O6qrq13bWrJkCc477zx06tQJeXl5OOKII/DFF1+4llu5ciVWrlyZdt8///xzxGIxXHfddcrjufbaa7FhwwZMnTo1w6OR2fud/bqXl43FYrjnnntw4IEHIi8vD3vuuSeOO+44jB07NuN9ISJq7XavrxWJiFoo+4P5nnvu6dwWj8cxZMgQHHfccXj00Uedls9rrrkGr7/+Oi6//HLccMMNWL16NZ555hnMmTMHkydPRnZ2NgDgzjvvxP33348zzjgDZ5xxBmbPno1TTz0V0Wg07f6MHTsWZ511Fnr06IEbb7wR3bt3x+LFi/HVV1/hxhtvxDXXXINNmzZh7NixeOutt9Kub+HChTj++OPRvn173HrrrcjOzsYLL7yAE088Ed9//z2OOuooZfm//e1v2GOPPXDXXXdhzZo1eOKJJ3D99dfjgw8+AJCsJDr11FPRpUsX3HbbbejYsSPWrFmDTz75xHc/3nrrLbz44ouYMWMGXn75ZQBwqkauuuoqvPHGGzjvvPPwj3/8A9OnT8fw4cOxePFifPrpp8p6li5diosvvhjXXHMNrr76ahx88MHG7UWjUQwZMgS1tbX429/+hu7du2Pjxo346quvUFJSgg4dOgBIjhN26KGH4pxzzkFWVha+/PJLXHfddbAsC3/961+Vda5YsQKXXHIJrrnmGvzpT3/Co48+irPPPhsjRozAHXfc4Vy4DR8+HBdccAGWLl2KcDj1vVkikcBpp52Go48+Gg8//DBGjRqFu+66C/F4HPfee6/nsSssLMTRRx/tXNx26dIF3377La688kqUlZUprYSXXnopvv/++0DhXKbKysoQjUZRVFSEN998EwsWLKjXWFTXXHMN9t57bzzwwAO44YYb8Jvf/AbdunVzfm96/QkhcM4552DChAm48sor0b9/f4wePRq33HILNm7ciMcff1zZxo8//ogvvvjCeQ6HDx+Os846C7feeiuee+45XHfdddixYwcefvhhXHHFFfjuu+989/kvf/kLPvroI1x//fXo27cvtm/fjkmTJmHx4sX49a9/jX/9618oLS3Fhg0bnH2xq/zKysrw8ssv4+KLL8bVV1+N8vJyvPLKKxgyZAhmzJiB/v3
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22 0.28\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNIAAAORCAYAAAA3ZI+fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd5hU1fkH8O+d2Ua30CyIqDEJRiXBaGzBgmAviRpN8hNrjDWK0aiJYkuwl9iwAdbYewEBQbqogNJ7L0vd3qbc3x+z9845555z587szC7Mfj/Pw8PuzJ3b7+zcd973PZZt2zaIiIiIiIiIiIjIV6ilV4CIiIiIiIiIiGhnwEAaERERERERERFRAAykERERERERERERBcBAGhERERERERERUQAMpBEREREREREREQXAQBoREREREREREVEADKQREREREREREREFwEAaERERERERERFRAAykERERERERERERBcBAGhFRK7Pvvvvi4osvdn+fMGECLMvChAkTsrYMy7Jw1113ZW1+2fbQQw9hv/32QzgcRp8+fVp6dXZIO/oxzJZvv/0WRx11FNq1awfLsjB79uyWXqUd1l133QXLslp6NfJea7n2iIiIdlYMpBERNaORI0fCsiz3X0lJCQ488EBce+21KC0tbenVS8vnn3++U97sffnll7jllltw9NFHY8SIEfjPf/7T0qvUaq1ZswZ33303Dj/8cOy6667o3LkzjjvuOIwdO9Yz7bhx43DppZfiwAMPRNu2bbHffvvh8ssvx4YNGzJefiQSwXnnnYdt27bhsccew6uvvoqePXs2ZZNa3BtvvIHHH38849fX1NTgrrvuympgPRssy8K1115rfP64447DL37xC+1zW7Zs8QSn1Pdiy7LQtWtXHH/88fjiiy/c6a6++mqEQiFs27ZNmue2bdsQCoVQXFyMuro66bnly5fDsizcfvvtGWzpzm9HPYeaWzwex4MPPohevXqhpKQEhxxyCP73v/8Fem0673fxeBzDhg1Dnz590L59e3Tr1g2nnHIKpk6dmu1NIiKiRgUtvQJERK3RPffcg169eqGurg6TJ0/Gs88+i88//xxz585F27Ztm3Vdfvvb36K2thZFRUVpve7zzz/H008/rQ2m1dbWoqBgx/wT89VXXyEUCuGll15Ke5spuz766CM88MADOPvsszFo0CBEo1G88sorOOmkkzB8+HBccskl7rT/+Mc/sG3bNpx33nn4yU9+guXLl+Opp57Cp59+itmzZ6N79+5pL3/ZsmVYtWoVXnjhBVx++eXZ3LQW88Ybb2Du3Lm44YYbMnp9TU0N7r77bgCJ4JToX//6F2699dYmruGOxXkvtm0bpaWlGDlyJE499VR88sknOP3003HMMcfg2WefxZQpU3DGGWe4r5s6dSpCoRAikQi+++47HHPMMe5zU6ZMAQDpsdbE7xxqTf75z3/i/vvvxxVXXIFf//rX+Oijj/DHP/4RlmXhggsu8H1tOu93N998Mx599FH8+c9/xtVXX42ysjI899xz6NevH6ZMmYLDDz8815tKRNTq7Jh3OUREee6UU07BYYcdBgC4/PLLsfvuu+PRRx/FRx99hAsvvFD7murqarRr1y7r6xIKhVBSUpLVeWZ7ftm0adMmtGnTJmtBNNu2UVdXhzZt2mRlfq3J8ccfj9WrV6Nz587uY3/961/Rp08f3HnnnVIg7dFHH8UxxxyDUCiZTH/yySejX79+eOqpp3DfffelvfxNmzYBAHbZZZeU0+bq+tuZFBQU7LAB8kyJ78UAcNlll6Fbt2743//+5wbSAGDy5MlSIG3KlCk45JBDUFtbi8mTJ0tBs8mTJyMUCuGoo45qvg2hHcq6devwyCOP4JprrsFTTz0FIPG3vl+/frj55ptx3nnnIRwOG18f9P0uGo3i2WefxbnnnotXX33Vnfa8887Dfvvth9dff52BNCKiHGBpJxHRDuCEE04AAKxYsQIAcPHFF6N9+/ZYtmwZTj31VHTo0AF/+tOfACTKOB5//HEcdNBBKCkpQbdu3XDllVdi+/bt0jxt28Z9992HvffeG23btsXxxx+PefPmeZZt6pH2zTff4NRTT8Wuu+6Kdu3a4ZBDDsETTzzhrt/TTz8NAFJplEPX42fWrFk45ZRT0LFjR7Rv3x4nnngipk+fLk3jlFtNmTIFgwcPRpcuXdCuXTucc8452Lx5szTtd999h4EDB6Jz585o06YNevXqhUsvvdR3P1uWhREjRqC6utpd55EjRwJI3JDce++92H///VFcXIx9990Xt99+O+rr66V57Lvvvjj99NMxevRoHHbYYWjTpg2ee+454zKXLFmC3//+9+jevTtKSkqw995744ILLkB5ebk7zYgRI3DCCSega9euKC4uRu/evfHss8965uUse8KECe6yDz74YPfYvf/++zj44INRUlKCvn37YtasWdLrnfNq+fLlGDhwINq1a4c999wT99xzD2zb9t13QOLm8NJLL0W3bt1QXFyMgw46CMOHD/dMt3r1aixcuDDl/A466CApiAYAxcXFOPXUU7F27VpUVla6j//2t7+Vbiqdx3bbbTcsWLAg5bJUF198Mfr16wcgcdNpWZabPeN3/VVXV+Omm25Cjx49UFxcjJ/+9Kd4+OGHPfvPKUV855130Lt3b7Rp0wZHHnkk5syZAwB47rnncMABB6CkpATHHXccVq5cmXKdKysrccMNN2DfffdFcXExunbtipNOOgkzZ84EkMj++eyzz7Bq1Sr3/N53330BAA0NDbjzzjvRt29fdOrUCe3atcOxxx6L8ePHu/NfuXIlunTpAgC4++673Xk417KuR1q6183kyZNx+OGHo6SkBPvttx9eeeWVlNvdnHbZZRe0adPGDRjus88+6NGjh5tl5pgyZQqOPvpoHHXUUdrnDjrooJQB2vr6etx4443o0qULOnTogDPPPBNr167VTpvq2istLUVBQYGbCSZatGgRLMtyAzomb775Jvr27YsOHTqgY8eOOPjgg933fEdZWRluuOEG9/w/4IAD8MADDyAejwNIfQ6lo6nX0KRJk3Deeedhn332QXFxMXr06IEbb7wRtbW1nmUtXLgQ5557LnbbbTeUlJTgsMMOw8cff+yZbtmyZVi2bFnKdf/oo48QiURw9dVXS9tz1VVXYe3atZg2bZrv64O+30UiEdTW1qJbt27StF27dkUoFJK+4En1/kFERMHl19eKREQ7KeeD+e677+4+Fo1GMXDgQBxzzDF4+OGH3ZLPK6+8EiNHjsQll1yC66+/HitWrMBTTz2FWbNmYcqUKSgsLAQA3Hnnnbjvvvtw6qmn4tRTT8XMmTMxYMAANDQ0pFyfMWPG4PTTT8cee+yBv/3tb+jevTsWLFiATz/9FH/7299w5ZVXYv369RgzZoz0LbjJvHnzcOyxx6Jjx4645ZZbUFhYiOeeew7HHXccvv76axxxxBHS9Ndddx123XVXDBkyBCtXrsTjjz+Oa6+9Fm+99RaARCbRgAED0KVLF9x6663YZZddsHLlSrz//vu+6/Hqq6/i+eefx4wZM/Diiy8CgJs1cvnll+Pll1/Gueeei5tuugnffPMNhg4digULFuCDDz6Q5rNo0SJceOGFuPLKK3HFFVfgpz/9qXZ5DQ0NGDhwIOrr63Hdddehe/fuWLduHT799FOUlZWhU6dOAIBnn30WBx10EM4880wUFBTgk08+wdVXX414PI5rrrlGmufSpUvxxz/+EVdeeSX+/Oc/4+GHH8YZZ5yBYcOG4fbbb3dv3IYOHYrzzz8fixYtkm7IYrEYTj75ZPzmN7/Bgw8+iFGjRmHIkCGIRqO45557jPuutLQUv/nNb9yb2y5duuCLL77AZZddhoqKCqmU8KKLLsLXX38dKDins3HjRrRt2zZlmXNVVRWqqqo8wbggrrzySuy11174z3/+g+uvvx6//vWvpZtR3fVn2zbOPPNMjB8/Hpdddhn69OmD0aNH4+abb8a6devw2GOPScuYNGkSPv74Y/cYDh06FKeffjpuueUWPPPMM7j66quxfft2PPjgg7j00kv
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17 0.23\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABMkAAAORCAYAAADoKvWAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd7wU1dkH8N/u3kbHQlUCSGwoYoIROxpRYjexoL5GRKMm9hYVEyVWNDYSG3aNsRCxK4LSVBABKSoISO8dbuH23T3vH3tn9pwzZ8ru3b2F/X0/H+Xe3dmZ2d3ZuXOefZ7nhIQQAkRERERERERERDks3Ng7QERERERERERE1NgYJCMiIiIiIiIiopzHIBkREREREREREeU8BsmIiIiIiIiIiCjnMUhGREREREREREQ5j0EyIiIiIiIiIiLKeQySERERERERERFRzmOQjIiIiIiIiIiIch6DZERERERERERElPMYJCMi2oX06NEDl156qf37lClTEAqFMGXKlIxtIxQK4R//+EfG1pdpjzzyCPbZZx9EIhEceuihjb07TVJTfw8zZdy4cTj00ENRVFSEUCiE4uLixt6lJuvSSy9Fjx49Gns3dmkrV65EKBTCq6++2ti7QkRERC4YJCMiypBXX30VoVDI/q+oqAj77bcfrr32WmzatKmxdy8lY8eObZZBlM8//xy33XYbjj76aLzyyit48MEHG3uXctqzzz6L8847D7/4xS8QCoWUAK7s+OOPVz478n/5+flpbXvbtm04//zz0aJFCzz99NN4/fXX0apVq3o8m8b3zDPP1CvAsn79evzjH//AvHnzMrZP9WUFjh599FHXZXr06IHTTz/deN93333nCDz94x//UI6hcDiMLl264PTTT8e3335rL3fqqadit912gxBCWefcuXMRCoXQvXt3x/YmTZqEUCiE559/PsVnumtoisdQY6iursbtt9+Orl27okWLFujfvz+++OKLQI997733MHjwYOyzzz5o2bIl9t9/f9xyyy3GIP5NN92EX//619h9993RsmVLHHjggfjHP/6BnTt3ZvgZERGRJa+xd4CIaFdz7733omfPnqiqqsLUqVPx7LPPYuzYsZg/fz5atmzZoPty3HHHobKyEgUFBSk9buzYsXj66aeNgbLKykrk5TXNPx+TJk1COBzGSy+9lPJzpsx7+OGHUVZWhsMPPxwbNmxwXe5vf/sb/vSnPym3lZeX489//jNOPvnktLY9a9YslJWV4b777sPAgQPTWkdT88wzz2DPPfd0DTb6Wb9+Pe655x706NHDkWX5wgsvIB6P138nm5Bnn30WrVu3Rjwex5o1a/DCCy/guOOOw8yZM3HooYfimGOOwWeffYb58+ejT58+9uOmTZuGvLw8rF69GmvXrsXee++t3AcAxxxzTIM/n6bA6xjKJZdeeinGjBmDG2+8Efvuuy9effVVnHrqqZg8ebLvsXHllVeia9euuPjii/GLX/wCP/74I5566imMHTsWc+bMQYsWLexlZ82ahWOPPRZDhw5FUVER5s6di4ceeggTJkzAV199hXCY+Q5ERJnWNEc5RETN2CmnnILDDjsMAPCnP/0Je+yxBx5//HF8+OGHuPDCC42PKS8vz0qWSzgcRlFRUUbXmen1ZdLmzZvRokWLjAXIhBCoqqpSBi0U3JdffmlnkbVu3dp1uZNOOslx23//+18AwP/93/+lte3NmzcDANq3b++7bEVFRYMHsJuadDP2mrJzzz0Xe+65p/372WefjYMPPhjvvPOOHSQDgKlTpzqCZKeeeiomTZqEqVOn4oILLrDvmzp1KvbYYw8ceOCBDfdEqEmZOXMm3n77bTzyyCO49dZbAQCXXHIJDj74YNx222345ptvPB8/ZswYHH/88cpt/fr1w5AhQ/DGG28oXxhMnTrV8fhevXrh1ltvxcyZM3HEEUfU/wkREZGCXz8QEWXZb3/7WwDAihUrACS+gW7dujWWLVuGU089FW3atLEDAfF4HCNHjsRBBx2EoqIidOrUCVdddRV27NihrFMIgfvvvx977703WrZsiRNOOAELFixwbNutJ9mMGTPsUqNWrVrhkEMOwb/+9S97/55++mkAUEqWLKZ+VnPnzsUpp5yCtm3bonXr1jjxxBOVsiYgWY46bdo03HzzzejQoQNatWqF3//+99iyZYuy7HfffYdBgwZhzz33RIsWLdCzZ09cdtllnq9zKBTCK6+8gvLycnufrRKsaDSK++67D7169UJhYSF69OiBO++8E9XV1co6rLKu8ePH47DDDkOLFi3w3HPPuW5zyZIlOOecc9C5c2cUFRVh7733xgUXXICSkhJ7mVdeeQW//e1v0bFjRxQWFqJ379549tlnHeuytj1lyhR723369LHfu/feew99+vRBUVER+vXrh7lz5yqPt46r5cuXY9CgQWjVqhW6du2Ke++911FOZrJu3Tpcdtll6NSpEwoLC3HQQQfh5Zdfdiy3evVqLFq0yHd9ANC9e3fl2EnFm2++iVatWuGss85K+bHHH388hgwZAgD4zW9+o5R6Hn/88Tj44IMxe/ZsHHfccWjZsiXuvPNOAInA2uWXX45OnTqhqKgIffv2xWuvvaasWy4PfPrpp+2SqZNPPhlr1qyBEAL33Xcf9t57b7Ro0QJnnXUWtm/f7rvPGzduxNChQ7H33nujsLAQXbp0wVlnnYWVK1cCSBwfCxYswJdffmkf39ZAe/v27bj11lvRp08ftG7dGm3btsUpp5yC77//3l7/lClT8Jvf/AYAMHToUMdnxNSTrLy8HLfccgu6deuGwsJC7L///nj00Ucdx1MoFMK1116LDz74AAcffLB9/IwbN873eTekzp07A4CdCXv44YejoKDAzg6zTJs2DccddxwOP/xw5b54PI5vv/0WRx11lO9xXVxcjEsvvRTt2rVD+/btMWTIENeeeIsWLcK5556L3XffHUVFRTjssMPw0Ucf2fdbJaX6sQgA48ePRygUwieffOK5P08++SQOOuggtGzZErvtthsOO+wwvPnmm8oyfucAv2MoqEx8hj788EOcdtpp6Nq1KwoLC9GrVy/cd999iMViju3NmDEDv/vd79CuXTu0bNkSAwYMcLznQOJ9WL16te/+jxkzBpFIBFdeeaV9W1FRES6//HJMnz4da9as8Xy8HiADgN///vcAgIULF/pu3/qcysdTWVkZbrzxRvTo0QOFhYXo2LEjTjrpJMyZM8d3fUREpGImGRFRli1btgwAsMcee9i3RaNRDBo0CMcccwweffRRO4vlqquuwquvvoqhQ4fi+uuvx4oVK/DUU09h7ty5mDZtmp3tcffdd+P+++/HqaeeilNPPRVz5szBySefjJqaGt/9+eKLL3D66aejS5cuuOGGG9C5c2csXLgQn3zyCW644QZcddVVWL9+Pb744gu8/vrrvutbsGABjj32WLRt2xa33XYb8vPz8dxzz+H444/Hl19+if79+yvLX3fdddhtt90wfPhwrFy5EiNHjsS1116L0aNHA0gEKk4++WR06NABd9xxB9q3b4+VK1fivffe89yP119/Hc8//zxmzpyJF198EQBw1FFHAUhk9L322ms499xzccstt2DGjBkYMWIEFi5ciPfff19Zz+LFi3HhhRfiqquuwhVXXIH999/fuL2amhoMGjQI1dXVuO6669C5c2esW7cOn3zyCYqLi9GuXTsAiZKvgw46CGeeeSby8vLw8ccf4+qrr0Y8Hsc111yjrHPp0qW46KKLcNVVV+Hiiy/Go48+ijPOOAOjRo3CnXfeiauvvhoAMGLECJx//vlYvHixUm4Ti8Xwu9/9DkcccQT++c9/Yty4cRg+fDii0Sjuvfde19du06ZNOOKII+xgR4cOHfDZZ5/h8ssvR2lpKW688UZ72UsuuQRffvlloMBburZs2YIvvvgCgwcPTivD8m9/+xv2339/PP/883b5c69evez7t23bhlNOOQUXXHABLr74YnTq1AmVlZU4/vjjsXTpUlx77bXo2bMn3nnnHVx66aUoLi7GDTfcoGzjjTfeQE1NDa677jp
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18 0.1\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNIAAAORCAYAAAA3ZI+fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd5QUVdoG8KfD5ECOioAYUZRdVBQDBhYUdXXX7LoCxs/sYsRdxcAu5hwwgroG1pyQHJScFRCQnGeAGSbPdKr6/ui+1ffequ6uGWZI8/zO4TDTXV1dXV3d0/X0e9/rMU3TBBERERERERERESXl3dsbQEREREREREREtD9gkEZEREREREREROQCgzQiIiIiIiIiIiIXGKQRERERERERERG5wCCNiIiIiIiIiIjIBQZpRERERERERERELjBIIyIiIiIiIiIicoFBGhERERERERERkQsM0oiIiIiIiIiIiFxgkEZERACATp06YeDAgdbvU6dOhcfjwdSpU+vtPjweDx599NF6W199e+aZZ3DooYfC5/Ohe/fue3tz9kn7+nNYX8aOHYvu3bsjMzMTHo8HJSUle3uT9lkDBw5Ep06d9vZmHNDWr18Pj8eDUaNG7e1NISIiavQYpBER7QNGjRoFj8dj/cvMzMQRRxyB22+/HYWFhXt782plzJgx+2XQMn78eNx///049dRTMXLkSPznP//Z25vUqL3xxhu47LLLcMghh8Dj8Sghr27BggW44IIL0LZtW+Tm5uK4447Dyy+/jEgkUqf7LioqwuWXX46srCy89tpr+PDDD5GTk1PHR7JveP3113crhNm6dSseffRRLF68uN62aXeJcOnZZ59NuEynTp1wwQUXOF43f/58Wzj16KOPKu/FXq8X7dq1wwUXXIDZs2dby/Xv3x/NmjWDaZrKOhctWgSPx4OOHTva7m/y5MnweDx46623avlIDwz74jG0NwQCATzwwANo3749srKy0LNnT0yYMMHVbVeuXIl//OMf6NWrlxXyr1+/vmE3mIiIbPx7ewOIiCju8ccfR+fOnVFTU4Pp06fjjTfewJgxY7B06VJkZ2fv0W0544wzUF1djfT09FrdbsyYMXjttdccw7Tq6mr4/fvmn57JkyfD6/Xi3XffrfVjpvr31FNPoby8HCeddBK2bduWcLkFCxagV69eOPzww/HAAw8gOzsbP/74I+666y6sWbMGL730Uq3ve968eSgvL8cTTzyBPn367M7D2Ge8/vrraNmyZdJAMpmtW7fiscceQ6dOnWzVmm+//TYMw9j9jdyHvPHGG8jNzYVhGNi0aRPefvttnHHGGZg7dy66d++O0047DT/++COWLl2Kbt26WbebMWMG/H4/Nm7ciM2bN+Pggw9WrgOA0047bY8/nn1BsmOoMRk4cCA+//xz3H333Tj88MMxatQo9O/fH1OmTEl5bMyaNQsvv/wyunbtiqOPPrrRh5JERHvLvnk2Q0TUSJ133nk44YQTAAA33HADWrRogeeffx7ffPMNrrrqKsfbVFZWNki1jNfrRWZmZr2us77XV5+2b9+OrKysegvRTNNETU0NsrKy6mV9jc20adOsarTc3NyEy7355psAgJ9++gnNmzcHANx8883o3bs3Ro0aVacgbfv27QCApk2bply2qqpqj4fc+5q0tLS9vQn17tJLL0XLli2t3y+++GIce+yx+Oyzz6wgDQCmT59uC9L69++PyZMnY/r06bjyyiut66ZPn44WLVrg6KOP3nMPhPYpc+fOxaeffopnnnkG9957LwDg2muvxbHHHov7778fM2fOTHr7P//5zygpKUFeXh6effZZBmlERHsJh3YSEe3Dzj77bADAunXrAES/yc7NzcWaNWvQv39/5OXl4W9/+xsAwDAMvPjiizjmmGOQmZmJNm3a4Oabb8auXbuUdZqmiWHDhuHggw9GdnY2zjrrLCxbtsx234l6pM2ZM8ca1pSTk4PjjjvOCisGDhyI1157DQCU4VGCU3+tRYsW4bzzzkN+fj5yc3NxzjnnKEOogPjQ1xkzZmDw4MFo1aoVcnJy8Je//AU7duxQlp0/fz769euHli1bIisrC507d8Z1112XdD97PB6MHDkSlZWV1jaL4V7hcBhPPPEEunTpgoyMDHTq1AkPPfQQAoGAsg4xhGzcuHE44YQTkJWVZYU8TlatWoVLLrkEbdu2RWZmJg4++GBceeWVKC0ttZYZOXIkzj77bLRu3RoZGRno2rUr3njjDdu6xH1PnTrVuu9u3bpZz92XX36Jbt26ITMzEz169MCiRYuU24vjau3atejXrx9ycnLQvn17PP7447aha062bNmC6667Dm3atEFGRgaOOeYYvPfee7blNm7ciBUrVqRcHwB07NhROXYSKSsrQ2Zmpi30ateuXZ1CzDPPPBMDBgwAAJx44onKsNIzzzwTxx57LBYsWIAzzjgD2dnZeOihhwBEw7frr78ebdq0QWZmJo4//ni8//77yrrloYivvfYaDj30UGRnZ6Nv377YtGkTTNPEE088gYMPPhhZWVm46KKLUFxcnHKbCwoKMGjQIBx88MHIyMhAu3btcNFFF1lDvjp16oRly5Zh2rRp1vF95plnAgCKi4tx7733olu3bsjNzUV+fj7OO+88/PLLL9b6p06dihNPPBEAMGjQINtrxKlHWmVlJe655x506NABGRkZOPLII/Hss8/ajiePx4Pbb78dX3/9NY499ljr+Bk7dmzKx70ntW3bFgCsitqTTjoJ6enpVpWZMGPGDJxxxhk46aSTlOsMw8Ds2bPRq1evlMd1SUkJBg4ciCZNmqBp06YYMGBAwh59K1aswKWXXormzZsjMzMTJ5xwAr799lvrejF8VT8WAWDcuHHweDz4/vvvk27PK6+8gmOOOQbZ2dlo1qwZTjjhBHz88cfKMqneA1IdQ27Vx2vom2++wfnnn4/27dsjIyMDXbp0wRNPPOE4FHzOnDk499xz0aRJE2RnZ6N379625xyIPg8bN25Muf2ff/45fD4fbrrpJuuyzMxMXH/99Zg1axY2bdqU9PbNmzdHXl5eyvsBgE8//RQ9evRAXl4e8vPz0a1btzp9sUBERHasSCMi2oetWbMGANCiRQvrsnA4jH79+uG0007Ds88+a1XD3HzzzRg1ahQGDRqEO++8E+vWrcOrr76KRYsWYcaMGVbVyCOPPIJhw4ahf//+6N+/PxYuXIi+ffsiGAym3J4JEybgggsuQLt27XDXXXehbdu2WL58Ob7//nvcdddduPnmm7F161ZMmDABH374Ycr1LVu2DKeffjry8/Nx//33Iy0tDW+++SbOPPNMTJs2DT179lSWv+OOO9CsWTMMHToU69evx4svvojbb78do0ePBhANM/r27YtWrVrhwQcfRNOmTbF+/Xp8+eWXSbfjww8/xFtvvYW5c+finXfeAQD06tULQLQy8P3338ell16Ke+65B3PmzMHw4cOxfPlyfPXVV8p6Vq5ciauuugo333wzbrzxRhx55JGO9xcMBtGvXz8EAgHccccdaNu2LbZs2YLvv/8eJSUlaNKkCYDo8LJjjjkGf/7zn+H3+/Hdd9/h1ltvhWEYuO2225R1rl69GldffTVuvvlmXHPNNXj22Wdx4YUXYsSIEXjooYdw6623AgCGDx+Oyy+/HCtXroTXG/8+LRKJ4Nxzz8XJJ5+Mp59+GmPHjsXQoUMRDofx+OOPJ9x3hYWFOPnkk61ApFWrVvjxxx9x/fXXo6ysDHfffbe17LXXXotp06a5CufcOvPMMzF69GjcfPPNGDx4sDW088svv8QzzzxT6/X985//xJFHHom33nrLGmrdpUsX6/qioiKcd955uPLKK3HNNdegTZs2qK6uxplnnonVq1fj9ttvR+fOnfHZZ59h4MCBKCkpwV133aXcx0cffYRgMIg77rgDxcXFePrpp3H55Zfj7LPPxtSpU/HAAw9g9erVeOWVV3Dvvfc6hpKySy65BMuWLcMdd9yBTp06Yfv27ZgwYQI2btyITp064cUXX8Qdd9yB3Nxc/POf/wQAtGnTBgCwdu1afP3
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "mean_mae = 0\n",
    "samples = []\n",
    "split = 'dev'\n",
    "station = station_with_worst_res_dev_set\n",
    "window='fixed'\n",
    "\n",
    "gen = train.get_data_generator(split=split, station=station , sampling_rate=sampling_rate, path=data_path, \n",
    "                                window='fixed')\n",
    "\n",
    "\n",
    "station_mae = []\n",
    "with torch.no_grad():\n",
    "    for i in range(len(gen)): \n",
    "        # idx = np.random.randint(len(gen))\n",
    "        idx = i\n",
    "        sample = gen[idx]\n",
    "        samples.append(sample)\n",
    "        pred = model(torch.tensor(sample[\"X\"], device=model.device).unsqueeze(0))  \n",
    "        pred = pred[0].cpu().numpy()\n",
    "\n",
    "        pred_pick_idx  = np.argmax(pred[0])\n",
    "        true_pick_idx = np.argmax(sample['y'][0])            \n",
    "            \n",
    "        mae_error = np.abs(pred_pick_idx - true_pick_idx) /100 #mae in seconds\n",
    "        station_mae.append(mae_error)\n",
    "\n",
    "sorted = np.argsort(station_mae)[::-1]\n",
    "mean_mae = np.mean(station_mae)\n",
    "print(mean_mae, station_with_worst_res_dev_set)\n",
    "print(np.array(station_mae)[sorted])\n",
    "\n",
    "\n",
    "## plot samples with mae error at leas 0.2s\n",
    "for idx in sorted:\n",
    "    if station_mae[idx] < 0.1: \n",
    "        break\n",
    "    print(idx, station_mae[idx])\n",
    "    plot_sample(samples[idx], model, idx, desc=f\" from station {station} {split} set\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fc074059-8faf-4d7b-8b2c-49995df6fb37",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test (2785, 17) 100\n",
      "test (2785, 17) 100\n",
      "test (2785, 17) 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3.11 2.33 1.23 1.21 0.59 0.53 0.39 0.23 0.22 0.12 0.11 0.11 0.1  0.1\n",
      " 0.1  0.09 0.09 0.08 0.08 0.08 0.07 0.07 0.07 0.06 0.06 0.06 0.06 0.06\n",
      " 0.05 0.05 0.05 0.04 0.04 0.04 0.04 0.04 0.04 0.04 0.04 0.04 0.04 0.04\n",
      " 0.04 0.04 0.04 0.04 0.04 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03\n",
      " 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03\n",
      " 0.03 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02\n",
      " 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02\n",
      " 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01\n",
      " 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.   0.   0.\n",
      " 0.   0.   0.   0.   0.   0.   0.   0.   0.  ]\n",
      "91 3.11\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNIAAAORCAYAAAA3ZI+fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd3gU5doG8Ht30ystEBCki4IURUVsiKJYsJxjOejxiFiODRuWIzYU+A52sSAgKmI7KvYKAlKkSRPpPXQSQkghPdmd74/NzL4z887s7G4quX/XxQWZzM7Mzs4umTvP874uRVEUEBERERERERERkS13XR8AERERERERERFRQ8AgjYiIiIiIiIiIyAEGaURERERERERERA4wSCMiIiIiIiIiInKAQRoREREREREREZEDDNKIiIiIiIiIiIgcYJBGRERERERERETkAIM0IiIiIiIiIiIiBxikEREREREREREROcAgjYioEerQoQNuueUW7ev58+fD5XJh/vz51bYPl8uFZ599ttq2V91eeukldOrUCR6PB3369Knrw6mX6vtrWF1mzpyJPn36IC4uDi6XC3l5eXV9SPXWLbfcgg4dOtT1YRARERHVGQZpRES17IMPPoDL5dL+xMXF4YQTTsCIESOQlZVV14cXkp9//rlBBi2//vorHnvsMZx99tmYNm0a/vvf/9b1ITVqWVlZGD58OFq2bIn4+HiceuqpmDFjhmm9LVu24KGHHsJZZ52lhV67du2KaN85OTm4/vrrER8fj4kTJ+Kjjz5CYmJiRNusa2+//TY++OCDsB9/4MABPPvss1izZk21HVOkdu3apfvcdLvdaNasGS699FIsXboUAFBRUYGePXuic+fOKCkpkW4jISEB1113HYDAZ3FcXBz2799vWv/888/HySefrFvWoUMH3XEkJibijDPOwIcffujoeRQXF+PZZ5+t1l9ayNTlZ3Ok19+x4KGHHsKpp56KZs2aISEhASeddBKeffZZFBYWOnr8pEmTcN111+H444+Hy+XS/eJLdPDgQTz++OMYOHAgkpOTq/0XYkREJBdV1wdARNRYjRkzBh07dkRpaSkWLVqESZMm4eeff8b69euRkJBQq8dy3nnnoaSkBDExMSE97ueff8bEiROlN2wlJSWIiqqf/8389ttvcLvdeO+990J+zlS9CgoKcM455yArKwsPPPAA0tPT8cUXX+D666/HJ598ghtvvFFbd+nSpXjjjTfQvXt3nHTSSdUS9KxYsQJHjx7F2LFjMWjQoIi3Vx+8/fbbaNGiheXNdzAHDhzAc889hw4dOpiqNadOnQqfzxf5QYbphhtuwGWXXQav14utW7fi7bffxsCBA7FixQr07NkT77zzDs4++2yMHTvWFJCPGDECMTExeOONN3TLy8rK8Pzzz+PNN990dAx9+vTBww8/DMAfZLz77rsYNmwYysrKcMcdd9g+tri4GM899xwAf1BXU+w+m2tapNffsWDFihU499xzMXz4cMTFxeHPP//E888/jzlz5mDhwoVwu+1rGV544QUcPXoUZ5xxBg4ePGi53pYtW/DCCy+ga9eu6NmzpxYqExFRzaqfdzhERI3ApZdeitNOOw0AcPvtt6N58+Z49dVX8d133+GGG26QPqaoqKhGqmXcbjfi4uKqdZvVvb3qdOjQIcTHx1dbiKYoCkpLSxEfH18t22tMpkyZgu3bt2Pu3Lm44IILAAB33303zjzzTDz88MO49tprtdfpyiuvRF5eHpKTk/Hyyy9XS5B26NAhAECTJk2CrltcXFzrIXd9Ex0dXaf7P/XUU3HTTTdpX5977rm49NJLMWnSJLz99tvo378/7rrrLrz88sv45z//iR49egAAvvrqK/z00094++230bp1a902+/Tpg6lTp2LUqFFo06ZN0GM47rjjdMdwyy23oFOnTnjttdeCBmnUOCxatMi0rHPnznjkkUewfPlynHnmmbaPX7BggVaNlpSUZLle3759kZOTg2bNmuHLL7/Uqi2JiKhmsbWTiKieUEOEjIwMAP6bs6SkJOzYsQOXXXYZkpOT8c9//hMA4PP5MGHCBPTo0QNxcXFo1aoV7rzzTuTm5uq2qSgKxo0bh7Zt2yIhIQEDBw7Ehg0bTPu2GiPtjz/+wGWXXYamTZsiMTERvXr1wuuvv64d38SJEwFA1+qkko2v9eeff+LSSy9FSkoKkpKScOGFF2LZsmW6ddR2q8WLF2PkyJFIS0tDYmIi/va3vyE7O1u37sqVKzF48GC0aNEC8fHx6NixI2699Vbb8+xyuTBt2jQUFRVpx6y2IVVWVmLs2LHo3LkzYmNj0aFDBzzxxBMoKyvTbaNDhw4YMmQIZs2ahdNOOw3x8fGYMmWK5T63bduGa665Bunp6YiLi0Pbtm0xdOhQ5Ofna+tMmzYNF1xwAVq2bInY2Fh0794dkyZNMm1L3ff8+fO1fffs2VN77b7++mv07NkTcXFx6Nu3L/7880/d49XraufOnRg8eDASExPRpk0bjBkzBoqi2J47ANi/fz9uvfVWtGrVCrGxsejRowfef/9903p79uzB5s2bg27v999/R1pamnb9A/5g9/rrr0dmZiYWLFigLW/WrBmSk5ODbtOp888/H8OGDQMAnH766boWKrWtb9WqVTjvvPOQkJCAJ554AoA/fLvtttvQqlUrxMXFoXfv3pg+fbpu22or4ssvv4yJEyeiU6dOSEhIwMUXX4y9e/dCURSMHTsWbdu2RXx8PK666iocOXIk6DFnZmZi+PDhaNu2LWJjY9G6dWtcddVVWotrhw4dsGHDBixYsEC7vtXKpyNHjuCRRx5Bz549kZSUhJSUFFx66aX466+/tO3Pnz8fp59+OgBg+PDhpveIbIy0oqIiPPzww2jXrh1iY2PRrVs3vPzyy6bryeVyYcSIEfj2229x8skna9fPzJkzgz5vK+eeey4AYMeOHdqy8ePHo0WLFrjrrrugKAoKCwvx4IMPaiGb0RNPPAGv14vnn38+rGNIS0vDiSeeqDsGmV27diEtLQ0A8Nxzz2nnVvyc3Lx5M6699lo0a9YMcXFxOO200/D999/rtlNRUYHnnnsOXbt2RVxcHJo3b45zzjkHs2fPBhD8s1nGyWepk/937K6/UKjvv7Vr12LAgAFISEhAly5d8OWXXwLwB039+vVDfHw8unXrhjlz5ugev3v3btxzzz3o1q0b4uPj0bx5c1x33XXSVvC8vDw8+OCD2vXbpUsXvPDCC6bKy4MHD2Lz5s2oqKgI+fkA0N43TsZgbN++fdDXDACSk5PRrFkzR/ufPXs2zjnnHDRp0gRJSUno1q2b9plGREShYUUaEVE9od6ENW/eXFtWWVmJwYMH45xzzsHLL7+sVcPceeed+OCDDzB8+HDcf//9yMjIwFtvvYU///wTixcv1qpGnnnmGYwbNw6XXXYZLrvsMqxevRoXX3wxysvLgx7P7NmzMWTIELRu3Vprudu0aRN+/PFHPPDAA7jzzjtx4MABzJ49Gx999FHQ7W3YsAHnnnsuUlJS8NhjjyE6OhpTpkzB+eefr90Uie677z40bdoUo0ePxq5duzBhwgSMGDECn3/+OQB/mHHxxRcjLS0Njz/+OJo0aYJdu3bh66+/tj2Ojz76CO+88w6WL1+Od999FwBw1llnAfBXBk6fPh3XXnstHn74Yfzxxx8YP348Nm3ahG+++Ua3nS1btuCGG27AnXfeiTvuuAPdunWT7q+8vByDBw9GWVkZ7rvvPqSnp2P//v348ccfkZeXh9TUVAD+MXF69OiBK6+8ElFRUfjhhx9wzz33wOfz4d5779Vtc/v27bjxxhtx55134qabbsLLL7+MK664ApMnT8YTTzyBe+65B4A/ULj++uuxZcsWXSuR1+vFJZdcgjPPPBMvvvgiZs6cidGjR6OyshJjxoyxPHdZWVk488wztUAkLS0Nv/zyC2677TYUFBTgwQcf1Na9+eabsWDBgqDhXFlZmbSST73WV61ahYsuush2G+F68skn0a1bN7zzzjtaq3Xnzp217+fk5ODSSy/F0KFDcdNNN6FVq1Y
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "46 2.33\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNIAAAORCAYAAAA3ZI+fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd5xU1f0+8GdmttJRekBATYwYlIRE7FiIYIsmsWB+fgWsUbGERKMmESyJNYoaFUFBNHYjMTaqYANEqnTpfZe6u2yddn5/zJ4755xb5s7ubOV5v17K7syde+/M3Jmd+8znfE5ACCFAREREREREREREnoINvQNERERERERERERNAYM0IiIiIiIiIiIiHxikERERERERERER+cAgjYiIiIiIiIiIyAcGaURERERERERERD4wSCMiIiIiIiIiIvKBQRoREREREREREZEPDNKIiIiIiIiIiIh8YJBGRERERERERETkA4M0IqJDUK9evTB8+HDr9zlz5iAQCGDOnDkZ20YgEMCYMWMytr5Me/zxx3HkkUciFAqhX79+Db07jVJjfw4z5bXXXsOPf/xjZGdno127dg29O43amWeeiTPPPLOhd4OIiIiowTBIIyKqZ6+88goCgYD1X15eHn70ox9h5MiRKCwsbOjdS8snn3zSJIOW6dOn46677sKpp56KSZMm4R//+EdD7xJV++qrr6zXxt69ex2Xefvtt3HyySejZcuWaNeuHU455RR89tlnNdremjVrMHz4cBx11FGYMGECxo8fX5vdbxT+8Y9/4L///W+Nb79q1SqMGTMGmzdvztg+1ZYM++V/oVAInTp1wqWXXorVq1cDAIqKitC1a1eceuqpEELY1jF//nwEg0HceeedAIAxY8YgEAigc+fOKC8vty3fq1cvXHjhhdpl6j4EAgG0adMGAwcOxMcff+zrfuzcuRNjxozB0qVL03wE0vPGG29g7NixdboNN7U9/pq6bdu24f7778eJJ56I9u3bo0OHDjjzzDMxc+ZMX7ffuXMnrrrqKhxzzDFo3bo12rVrhxNPPBGTJ0+2HddTpkzB4MGD0a1bN+Tm5qJ79+649NJLsWLFirq4a0REVC2roXeAiOhQ9cADD6B3796orKzEV199hRdeeAGffPIJVqxYgRYtWtTrvpxxxhmoqKhATk5OWrf75JNP8NxzzzmGaRUVFcjKapx/Zj777DMEg0G8/PLLad9nqjvxeBy33norWrZsibKyMsdlxowZgwceeACXXnophg8fjkgkghUrVmDHjh012uacOXMQj8fx9NNP4+ijj67N7jca//jHP3DppZfikksuqdHtV61ahfvvvx9nnnkmevXqpV03ffr02u9gLdx22234xS9+gUgkgu+++w7jxo3DnDlzsGLFCnTp0gVjx47F0KFDMWHCBNxwww3W7aLRKH7/+9+jZ8+euP/++7V17t69Gy+88AL++Mc/+tqHX/7yl7j66qshhMCWLVvwwgsv4KKLLsKnn36KwYMHe952586duP/++9GrV686rYR94403sGLFCtxxxx11tg03tT3+mroPPvgAjz76KC655BIMGzYM0WgUr776Kn75y19i4sSJGDFihOft9+7di+3bt+PSSy/FEUccgUgkghkzZmD48OFYu3at9sXP8uXL0b59e9x+++3o0KEDCgoKMHHiRJx44omYN28eTjjhhLq+u0REh6TGeYZDRHQIOO+88/Dzn/8cAHDdddfh8MMPx5NPPokPPvgAV155peNtysrK0LJly4zvSzAYRF5eXkbXmen1ZdLu3buRn5+fsRBNCIHKykrk5+dnZH2HqvHjx2Pbtm247rrr8PTTT9uunz9/Ph544AH885//xB/+8IeMbHP37t0AkHJIJ5/jhIYOnk8//XRceuml1u/HHHMMbrrpJrz66qu46667cMUVV2Dy5Mm4++67cfHFF6Nz584AgKeffhrLli3DJ598Yvuiol+/fnj88cdx8803+3p+f/SjH+Gqq66yfv/tb3+LPn364Omnn04ZpFHzd9ZZZ2Hr1q3o0KGDddnvf/979OvXD/fdd1/KIO3444+3tVkYOXIkLrroIjzzzDN48MEHEQqFAAD33Xef7fbXXXcdunfvjhdeeAHjxo2r/R0iIiIbDu0kImokzj77bADApk2bAADDhw9Hq1atsGHDBpx//vlo3bo1/t//+38AEpU7Y8eOxXHHHYe8vDx07twZN954Iw4cOKCtUwiBhx56CN27d0eLFi1w1llnYeXKlbZtu/VI++abb3D++eejffv2aNmyJY4//ngr4Bg+fDiee+45APpwJ8mpv9aSJUtw3nnnoU2bNmjVqhXOOecczJ8/X1tGDn39+uuvMWrUKHTs2BEtW7bEr3/9a+zZs0dbduHChRg8eDA6dOiA/Px89O7dG9dcc43n4xwIBDBp0iSUlZVZ+/zKK68ASFStPPjggzjqqKOQm5uLXr164d5770VVVZW2Djnka9q0afj5z3+O/Px8vPjii67bXLduHX7729+iS5cuyMvLQ/fu3TF06FAUFxdby0yaNAlnn302OnXqhNzcXPTp0wcvvPCCbV1y23PmzLG23bdvX+u5e//999G3b1/k5eWhf//+WLJkiXZ7eVxt3LgRgwcPRsuWLdGtWzc88MADjsPhTDt27MA111yDzp07Izc3F8cddxwmTpxoW27r1q1Ys2ZNyvVJ+/fvx1//+lc88MADrqHW2LFj0aVLF9x+++0QQqC0tNT3+p306tULo0ePBgB07NhRO2a9nuONGzfisssuw2GHHYYWLVrgpJNOsg3tk6+pd955B/fffz9+8IMfoHXr1rj00ktRXFyMqqoq3HHHHejUqRNatWqFESNG2I4zJ6mOpUAggLKyMkyePNk6vmU/xC1btuDmm2/GMcccg/z8fBx++OG47LLLtCGcr7zyCi677DIAiUBArkMeX0490nbv3o1rr70WnTt3Rl5eHk444QRMnjxZW2bz5s0IBAJ44oknMH78eOs19otf/ALffvttyvvt5vTTTwcAbNiwwbrs+eefR1VVFUaNGgUgMdRuzJgxuOKKK3DeeefZ1nHfffehsLDQ8fXmx7HHHosOHTpo++Bkzpw5+MUvfgEAGDFihO39B0i85w4ZMgRt27ZFixYtMHDgQHz99dfaeg4ePIg77rgDvXr1Qm5uLjp16oRf/vKXWLx4MYDEc/Txxx9jy5Yt1jbMykLTjBkzcNppp6Fdu3Zo1aoVjjnmGNx7773aMlVVVRg9ejSOPvpo5ObmokePHrjrrru049br+EtHbd/nvvvuOwwfPhxHHnkk8vLy0KVLF1xzzTXYt2+fbVuZfk877rjjtBANAHJzc3H++edj+/btOHjwYBqPRFKvXr1QXl6OcDjsuVynTp3QokULFBUVaZe/9dZb6N+/P1q3bo02bdqgb9++jl9YEBFRaqxIIyJqJORJ2OGHH25dFo1GMXjwYJx22ml44oknrEqKG2+8Ea+88gpGjBiB2267DZs2bcK//vUvLFmyBF9//TWys7MBJE4QH3roIZx//vk4//zzsXjxYpx77rkpP4gDiROrCy+8EF27dsXtt9+OLl26YPXq1fjoo49w++2348Ybb8TOnTsxY8YMvPbaaynXt3LlSpx++ulo06YN7rrrLmRnZ+PFF1/EmWeeic8//xwDBgzQlr/11lvRvn17jB49Gps3b8bYsWMxcuRIvP322wASJ+/nnnsuOnbsiLvvvhvt2rXD5s2b8f7773vux2uvvYbx48djwYIFeOmllwAAp5xyCoDEN/mTJ0/GpZdeij/+8Y/45ptv8PDDD2P16tWYMmWKtp61a9fiyiuvxI033ojrr78exxxzjOP2wuEwBg8ejKqqKtx6663o0qULduzYgY8++ghFRUVo27YtAOCFF17Acccdh1/96lfIysrChx9+iJtvvhnxeBy33HKLts7169fjd7/7HW688UZcddVVeOKJJ3DRRRdh3LhxuPfee3HzzTcDAB5++GFcfvnlWLt2LYLB5HdnsVgMQ4YMwUknnYTHHnsMU6dOxejRoxGNRvHAAw+4PnaFhYU
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63 1.23\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNIAAAORCAYAAAA3ZI+fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd5hTZdoG8DvJ9Bk6Q0e6DRF2UbGDimLBsp9lcVdFXBULltXVFRuKunbFCoqKfS3YVlGaUqWLSBPpnYGhDNMn7Xx/JO/Je95TcjKTmQxw/67Li5lMcs7JyclIbp7neT2apmkgIiIiIiIiIiIiR95UHwAREREREREREdGBgEEaERERERERERGRCwzSiIiIiIiIiIiIXGCQRkRERERERERE5AKDNCIiIiIiIiIiIhcYpBEREREREREREbnAII2IiIiIiIiIiMgFBmlEREREREREREQuMEgjIiIiIiIiIiJygUEaEdEhqGPHjrj22mv176dPnw6Px4Pp06cnbR8ejwePPPJI0raXbM8++yw6d+4Mn8+HXr16pfpw6qX6/homy8SJE9GrVy9kZWXB4/GgqKgo1YdUb1177bXo2LFjqg+DiIiIKGUYpBER1bF3330XHo9H/y8rKwuHH344hg0bhp07d6b68BLy/fffH5BBy+TJk3HvvffilFNOwbhx4/Cf//wn1Yd0yNu5cyeGDh2Ktm3bIisrCx07dsQ//vEPw32++uorDBgwAG3atEFmZibatWuHyy67DMuXL6/2fvfs2YMrrrgC2dnZeO211/DBBx8gNze3pk8npV5//XW8++671X789u3b8cgjj2DJkiVJO6aa2rhxo+H3ptfrRdOmTXHeeedh7ty5AIBAIIAePXqgS5cuqKiosNxGTk4OLr/8cgCx38VZWVnYtm2b6f79+vXDMcccY7itY8eOhuPIzc3FCSecgPfff9/V8ygvL8cjjzyS1H+0sJLK3801vf4OBp9++imuuuoqdOvWDR6PB/369XP92C1btuDRRx/FCSecgCZNmqB58+bo168fpk6darrvzJkzcdFFF6F9+/bIyspCq1atcO655+Lnn39O4rMhIiJVWqoPgIjoUDVy5Eh06tQJlZWVmD17NkaPHo3vv/8ey5cvR05OTp0ey+mnn46KigpkZGQk9Ljvv/8er732muUHtoqKCqSl1c//zfz000/wer14++23E37OlHxbtmzBKaecAgC46aab0LZtW2zfvh0LFiww3G/ZsmVo0qQJ7rjjDjRv3hwFBQV45513cMIJJ2Du3Lno2bNnwvteuHAhSkpK8Nhjj6F///5JeT6p9vrrr6N58+aGqtNEbN++HY8++ig6duxoqtYcO3YswuFwzQ+ymq688kqcf/75CIVCWL16NV5//XWcccYZWLhwIXr06IE333wTp5xyCh577DFTQD5s2DBkZGTg5ZdfNtxeVVWFp556Cq+88oqrY+jVqxfuvvtuAMCOHTvw1ltvYfDgwaiqqsINN9zg+Njy8nI8+uijAJBQuJIop9/Nta2m19/BYPTo0fjll19w/PHHY8+ePQk99ptvvsHTTz+NSy65BIMHD0YwGMT777+Ps88+G++88w6GDBmi33f16tXwer246aab0KpVK+zbtw8ffvghTj/9dEyYMAHnnntusp8aERGBQRoRUcqcd955OO644wAA119/PZo1a4YXXngB33zzDa688krLx5SVldVKtYzX60VWVlZSt5ns7SXTrl27kJ2dnbQQTdM0VFZWIjs7OynbO9QMHToUaWlpWLhwIZo1a2Z7v4cffth02/XXX4927dph9OjRGDNmTML73rVrFwCgcePGce9bXl5e5yF3fZOenp7S/f/5z3/GVVddpX9/2mmn4bzzzsPo0aPx+uuv46STTsJNN92E5557Dn//+9/RvXt3AMAXX3yBCRMm4PXXX0fr1q0N2+zVqxfGjh2L4cOHo02bNnGPoW3btoZjuPbaa9G5c2e8+OKLcYM0OjR88MEHaNu2Lbxer6mqMZ4zzjgDmzdvRvPmzfXbbrrpJvTq1QsPP/ywIUi7/vrrcf311xsef8stt6Bz584YNWoUgzQiolrC1k4ionrizDPPBABs2LABQOTDWV5eHtatW4fzzz8fDRo0wN///ncAQDgcxqhRo9C9e3dkZWWhZcuWGDp0KPbt22fYpqZpePzxx9GuXTvk5OTgjDPOwIoVK0z7tpuRNn/+fJx//vlo0qQJcnNzceyxx+Kll17Sj++1114DAEOrk2A1X+vXX3/Feeedh4YNGyIvLw9nnXUW5s2bZ7iPaLf6+eefcddddyE/Px+5ubn4y1/+gsLCQsN9Fy1ahAEDBqB58+bIzs5Gp06dcN111zmeZ4/Hg3HjxqGsrEw/ZtGGFAwG8dhjj6FLly7IzMxEx44dcf/996OqqsqwjY4dO2LgwIGYNGkSjjvuOGRnZ+ONN96w3eeaNWtw6aWXolWrVsjKykK7du0waNAg7N+/X7/PuHHjcOaZZ6JFixbIzMzE0UcfjdGjR5u2JfY9ffp0fd89evTQX7svv/wSPXr0QFZWFnr37o1ff/3V8HhxXa1fvx4DBgxAbm4u2rRpg5EjR0LTNMdzBwDbtm3Dddddh5YtWyIzMxPdu3fHO++8Y7rf5s2bsWrVqrjbW7VqFX744Qfcc889aNasGSorKxEIBOI+TmjRogVycnKqNdesX79+GDx4MADg+OOPh8fj0atoRFvfL7/8gtNPPx05OTm4//77AUTCt3/84x9o2bIlsrKy0LNnT7z33nuGbYtWxOeeew6vvfYaOnfujJycHJxzzjnYsmULNE3DY489hnbt2iE7OxsXX3wx9u7dG/eYCwoKMGTIELRr1w6ZmZlo3bo1Lr74YmzcuBFA5PpYsWIFZsyYoV/fovJp7969+Ne//oUePXogLy8PDRs2xHnnnYfffvtN3/706dNx/PHHAwCGDBlieo9YzUgrKyvD3Xffjfbt2yMzMxNHHHEEnnvuOdP15PF4MGzYMHz99dc45phj9Otn4sSJcZ+3ndNOOw0AsG7dOv22J598Es2bN8dNN90ETdNQWlqKO++8Uw/ZVPfffz9CoRCeeuqpah1Dfn4+jjzySMMxWNm4cSPy8/MBAI8++qh+buXfk6tWrcJll12Gpk2bIisrC8cddxz+97//GbYTCATw6KOPolu3bsjKykKzZs1w6qmnYsqUKQDi/2624uZ3qZv/7zhdf4kQ77+lS5eib9++yMnJQdeuXTF+/HgAwIwZM9CnTx9kZ2fjiCOOMLU9btq0CbfccguOOOIIZGdno1mzZrj88sv194msqKgId955p379du3aFU8//bSp8nLHjh1YtWqVq99P7du3h9dbvY9Z3bt3N4RoAJCZmYnzzz8fW7duRUlJiePjc3JykJ+fb/qdOGXKFJx66qlo3Lgx8vLycMQRR+i/04iIKDGsSCMiqifEhzC5IicYDGLAgAE49dRT8dxzz+nVMEOHDsW7776LIUOG4Pbbb8eGDRvw6quv4tdff8XPP/+sV408/PDDePzxx3H++efj/PPPx+LFi3HOOefA7/fHPZ4pU6Zg4MCBaN26Ne644w60atUKv//+O7777jvccccdGDp0KLZv344pU6bggw8+iLu9FStW4LTTTkPDhg1x7733Ij09HW+88Qb69eunfyiS3XbbbWjSpAlGjBiBjRs3YtSoURg2bBg+/fRTAJEw45xzzkF+fj7uu+8+NG7cGBs3bsSXX37peBwffPAB3nzzTSxYsABvvfUWAODkk08GEPnX/ffeew+XXXYZ7r77bsyfPx9PPvkkfv/9d3z11VeG7fzxxx+48sorMXToUNxwww044ogjLPfn9/sxYMAAVFVV4bbbbkOrVq2wbds2fPfddygqKkKjRo0ARFqBunfvjosuughpaWn49ttvccsttyAcDuPWW281bHPt2rX429/+hqFDh+Kqq67Cc889hwsvvBBjxozB/fffj1tuuQVAJFC44oor8Mcffxg+1IVCIZx77rk48cQT8cwzz2DixIkYMWIEgsEgRo4caXvudu7ciRNPPFEPRPL
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40 1.21\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNIAAAORCAYAAAA3ZI+fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd5wU5f0H8M/s7jXK0asQQDQqBiViRGLBQkRFo8aGJj8FuwSNIdGfmghYEuxiR1ARjcaO/GwUEVCkSROp0vsd9Xrd3fn9sfvMPPNM2dm93TvAzzsvw93u7Mzs3t7BfO77fb6arus6iIiIiIiIiIiIyFOgoU+AiIiIiIiIiIjoUMAgjYiIiIiIiIiIyAcGaURERERERERERD4wSCMiIiIiIiIiIvKBQRoREREREREREZEPDNKIiIiIiIiIiIh8YJBGRERERERERETkA4M0IiIiIiIiIiIiHxikERERERERERER+cAgjYjoZ6hr164YPHiw8fmsWbOgaRpmzZqVtmNomoZRo0albX/p9sQTT+DII49EMBhEr169Gvp0DkoH+9cwXd566y0ce+yxyMrKQvPmzRv6dA5qZ511Fs4666yGPg0iIiKiBsMgjYionr3xxhvQNM34Lzc3F7/85S8xbNgwFBYWNvTpJeWLL744JIOWadOm4Z577sFpp52GCRMm4N///ndDnxLFzZkzx/je2Lt3r+3+HTt24KqrrkLz5s2Rn5+PSy65BBs3bkz5eGvWrMHgwYPRvXt3jB8/HuPGjavL6R8U/v3vf+OTTz5J+fGrVq3CqFGjsHnz5rSdU12JsF/8FwwG0bZtW1xxxRVYvXo1AKCoqAgdOnTAaaedBl3XbfuYP38+AoEA7r77bgDAqFGjoGka2rVrh4qKCtv2Xbt2xUUXXWS5TT4HTdOQn5+Pfv364fPPP/f1PHbu3IlRo0Zh2bJlSb4CyXnnnXcwZsyYjB7DTV3ff4eDl19+GVdeeSV+8YtfQNM0yy+uElmzZg3uuece9OrVC02bNkWHDh0wcOBALFq0yLbt2rVr8de//hW//e1vkZubC03TDqrvWyKiw1WooU+AiOjn6qGHHkK3bt1QVVWFOXPm4OWXX8YXX3yBFStWoFGjRvV6LmeeeSYqKyuRnZ2d1OO++OILvPjii45hWmVlJUKhg/Ovma+//hqBQACvvfZa0s+ZMicajeKOO+5A48aNUV5ebru/rKwMZ599NoqLi3H//fcjKysLzzzzDPr164dly5ahVatWSR9z1qxZiEajePbZZ3HUUUel42k0uH//+9+44oorcOmll6b0+FWrVuHBBx/EWWedha5du1rumzZtWt1PsA7uvPNO/OY3v0FtbS2WL1+OsWPHYtasWVixYgXat2+PMWPGYNCgQRg/fjxuueUW43HhcBi33XYbunTpggcffNCyz927d+Pll1/G3/72N1/n8Lvf/Q7XXXcddF3Hli1b8PLLL+Piiy/Gl19+iQEDBng+dufOnXjwwQfRtWvXjFbCvvPOO1ixYgXuuuuujB3DTV3ff4eDxx57DKWlpTjllFOwa9eupB776quv4rXXXsPll1+OoUOHori4GK+88gpOPfVUTJkyBf379ze2nTdvHp577jn06NEDxx13XMYDWiIiijk4r3CIiH4GLrjgApx88skAgJtuugmtWrXC008/jcmTJ+Oaa65xfEx5eTkaN26c9nMJBALIzc1N6z7Tvb902r17N/Ly8tIWoum6jqqqKuTl5aVlfz9X48aNw7Zt23DTTTfh2Weftd3/0ksvYd26dVi4cCF+85vfAIh9H/3qV7/CU089lVJl4e7duwEgYUsnv8YxDR08n3HGGbjiiiuMz4855hjcfvvtePPNN3HPPffg6quvxsSJE3HvvffikksuQbt27QAAzz77LH744Qd88cUXtl9U9OrVC0888QSGDh3q6+v7y1/+En/605+Mzy+//HL06NEDzz77bMIgjX4eZs+ebVSjNWnSJKnHXnPNNRg1apTlcTfccAOOO+44jBo1yhKk/f73v0dRURGaNm2KJ598kkEaEVE9YWsnEdFB4pxzzgEAbNq0CQAwePBgNGnSBBs2bMCFF16Ipk2b4o9//COAWOXOmDFjcPzxxyM3Nxft2rXDrbfeigMHDlj2qes6HnnkEXTq1AmNGjXC2WefjZUrV9qO7bZG2oIFC3DhhReiRYsWaNy4MU444QQj4Bg8eDBefPFFANZ2J8Fpfa2lS5figgsuQH5+Ppo0aYJzzz0X8+fPt2wjWl+/++47DB8+HG3atEHjxo1x2WWXYc+ePZZtFy1ahAEDBqB169bIy8tDt27dcMMNN3i+zpqmYcKECSgvLzfO+Y033gAQq1p5+OGH0b17d+Tk5KBr1664//77UV1dbdmHaPmaOnUqTj75ZOTl5eGVV15xPea6detw+eWXo3379sjNzUWnTp0waNAgFBcXG9tMmDAB55xzDtq2bYucnBz06NEDL7/8sm1f4tizZs0yjt2zZ0/ja/fxxx+jZ8+eyM3NRe/evbF06VLL48X7auPGjRgwYAAaN26Mjh074qGHHnJsh1Pt2LEDN9xwA9q1a4ecnBwcf/zxeP31123bbd26FWvWrEm4P2H//v345z//iYceesg11Prwww/xm9/8xgjRAODYY4/Fueeei/fff9/3sYSuXbti5MiRAIA2bdpY3rNeX+ONGzfiyiuvRMuWLdGoUSOceuqpttY+8T31/vvv48EHH8QRRxyBpk2b4oorrkBxcTGqq6tx1113oW3btmjSpAmGDBlie585SfRe0jQN5eXlmDhxovH+Fm1lW7ZswdChQ3HMMccgLy8PrVq1wpVXXmlpBXvjjTdw5ZVXAgDOPvtsYx/i/eW0Rtru3btx4403ol27dsjNzcWJJ56IiRMnWrbZvHkzNE3Dk08+iXHjxhnfY7/5zW/w/fffJ3zebs444wwAwIYNG4zbXnrpJVRXV2P48OEAgG3btmHUqFG4+uqrccEFF9j2MWLECBQWFjp+v/lx3HHHoXXr1pZzcDJr1izjvTtkyBDbzx8g9jP3/PPPR7NmzdCoUSP069cP3333nWU/paWluOuuu9C1a1fk5OSgbdu2+N3vfoclS5YAiH2NPv/8c2zZssU4hlpZqJo+fTpOP/10NG/eHE2aNMExxxyD+++/37JNdXU1Ro4ciaOOOgo5OTno3Lkz7rnnHsv71uv9l4y6/pxbvnw5Bg8ejCOPPBK5ublo3749brjhBuzbt892rEz8TOvSpYvl78Nk9O7d2xa+tWrVCmeccYbRxiy0bNkSTZs29bXfd999F71790bTpk2Rn5+Pnj17Ov7CgoiIEmNFGhHRQUJchMntaeFwGAMGDMDpp5+OJ5980qikuPXWW/HGG29gyJAhuPPOO7Fp0ya88MILWLp0Kb777jtkZWUBiF0gPvLII7jwwgtx4YUXYsmSJTjvvPNQU1OT8HymT5+Oiy66CB06dMBf/vIXtG/fHqtXr8Znn32Gv/zlL7j11luxc+dOTJ8+HW+99VbC/a1cuRJnnHEG8vPzcc899yArKwuvvPIKzjrrLMyePRt9+vSxbH/HHXegRYsWGDlyJDZv3owxY8Zg2LBheO+99wDELt7PO+88tGnTBvfeey+aN2+OzZs34+OPP/Y8j7feegvjxo3DwoUL8eqrrwIAfvvb3wKIVQZOnDgRV1xxBf72t79hwYIFGD16NFavXo1JkyZZ9rN27Vpcc801uPXWW3HzzTfjmGOOcTxeTU0NBgwYgOrqatxxxx1o3749duzYgc8++wxFRUVo1qwZgNiaOscffzx+//vfIxQK4dNPP8XQoUMRjUbx5z//2bLP9evX49prr8Wtt96KP/3pT3jyySdx8cUXY+zYsbj//vsxdOhQAMDo0aNx1VVXYe3atQgEzN+dRSIRnH/++Tj11FPx+OOPY8qUKRg5ciTC4TAeeugh19eusLAQp556KjRNw7Bhw9CmTRt8+eWXuPHGG1FSUmJpI7vuuuswe/ZsX+EcADzwwANo3749br31Vjz88MO2+6PRKJYvX+4YlJ5yyimYNm0aSkt
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28 0.59\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNIAAAORCAYAAAA3ZI+fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOydd5wV1f3+n3vvNliaCooFBUs0GpR8ScQWQSVijeYXNSYxIrHFbjQaNYkNDLYoFgQrlthii50qqCgKgo0qvXdYlt1ld285vz/uPTPnnDkzd25ZdoHn/Xop986dOXNm5szdOc99Pp9PRAghQAghhBBCCCGEEEIICSTa3B0ghBBCCCGEEEIIIWRbgEIaIYQQQgghhBBCCCEhoJBGCCGEEEIIIYQQQkgIKKQRQgghhBBCCCGEEBICCmmEEEIIIYQQQgghhISAQhohhBBCCCGEEEIIISGgkEYIIYQQQgghhBBCSAgopBFCCCGEEEIIIYQQEgIKaYQQQgghhBBCCCGEhIBCGiGE7IB07doVF1xwgfN+woQJiEQimDBhQtH2EYlEcPvttxetvWJz3333Yd9990UsFkOPHj2auzstkpZ+DYvFyJEj0aNHD1RUVCASiaCqqqq5u9RiueCCC9C1a9fm7gYhhBBCSLNBIY0QQrYyzz77LCKRiPNfRUUFfvSjH+HKK6/E6tWrm7t7OfHBBx9sk0LL6NGjceONN+Loo4/GiBEj8K9//au5u7TDsnTpUtxxxx04/PDDsdNOO6Fjx47o06cPxo4da11/6tSpOO2009C5c2e0adMGhx56KB5++GEkk8m89r9+/Xqcc845aNWqFYYOHYoXXngBlZWVhRxSs/PYY4/h2WefzXv7FStW4Pbbb8c333xTtD4VyqJFi7TvzWg0ip133hknn3wyJk2aBACIx+Po3r079ttvP2zZssXaRuvWrXH22WcDcL+LKyoqsHz5cs/6ffr0wU9+8hNtWdeuXbV+VFZW4vDDD8fzzz8f6jjq6upw++23F/VHCxvN+d1c6PjbXnjnnXfwf//3f6ioqMDee++N2267DYlEIut25lhX/3vllVc86z/66KP48Y9/jPLycuy555647rrrUFtb2xSHRAghJENJc3eAEEJ2VO68805069YN9fX1mDhxIoYNG4YPPvgA06dPR+vWrbdqX4499lhs2bIFZWVlOW33wQcfYOjQodYJ25YtW1BS0jL/zHz00UeIRqN4+umncz5mUlzefvtt3HPPPTjzzDPRv39/JBIJPP/88/jlL3+JZ555BgMGDHDWnTp1Ko466igccMAB+Nvf/obWrVvjww8/xDXXXIP58+fjoYceynn/U6ZMwebNmzFw4ED07du3mIfWbDz22GPo2LGj5jrNhRUrVuCOO+5A165dPW7NJ598EqlUqvBO5snvfvc7nHLKKUgmk/jhhx/w2GOP4bjjjsOUKVPQvXt3PPHEEzj66KMxcOBAj0B+5ZVXoqysDA8//LC2vKGhAXfffTceeeSRUH3o0aMHrr/+egDAypUr8dRTT6F///5oaGjAxRdfHLhtXV0d7rjjDgBpoa6pCPpubmoKHX/bAx9++CHOPPNM9OnTB4888gi+//57DBo0CGvWrMGwYcNCtSHHusqRRx6pvf/b3/6Ge++9F2eddRauueYazJw5E4888ghmzJiBUaNGFe14CCGE6LTMGQ4hhOwAnHzyyfjZz34GALjooouwyy674IEHHsDbb7+N3/3ud9Ztamtrm8QtE41GUVFRUdQ2i91eMVmzZg1atWpVNBFNCIH6+nq0atWqKO3tSBx33HFYsmQJOnbs6Cz785//jB49euDWW2/VhLTHH38cAPDJJ59g5513BgBceuml6N27N5599tm8hLQ1a9YAADp06JB13bq6uq0ucrc0SktLm3X///d//4fzzjvPef+LX/wCJ598MoYNG4bHHnsMRx55JP785z/j/vvvxx/+8AcccsghAIA33ngD77//Ph577DHsvvvuWps9evTAk08+iZtvvhl77LFH1j7sueeeWh8uuOAC7LvvvnjwwQezCmlkx+Cvf/0rDj30UIwePdr5Qaldu3b417/+hWuuuQYHHXRQ1jbMsW6ycuVKPPDAA/jjH/+oOSJ/9KMf4aqrrsK7776L008/vfCDIYQQ4oGhnYQQ0kI4/vjjAQALFy4EkJ6ctWnTBvPnz8cpp5yCtm3b4g9/+AMAIJVKYciQITjkkENQUVGB3XbbDZdeeik2btyotSmEwKBBg7DXXnuhdevWOO644zBjxgzPvv1ypH355Zc45ZRTsNNOO6GyshKHHnqoI1ZccMEFGDp0KABooScSW36tr7/+GieffDLatWuHNm3a4IQTTsAXX3yhrSPDrT777DNcd9116NSpEyorK/HrX/8aa9eu1db96quv0K9fP3Ts2BGtWrVCt27d8Kc//SnwPEciEYwYMQK1tbVOn2UYUiKRwMCBA7HffvuhvLwcXbt2xS233IKGhgatja5du+K0007DqFGj8LOf/QytWrVyRB4bc+fOxW9+8xt07twZFRUV2GuvvXDuuedi06ZNzjojRozA8ccfj1133RXl5eU4+OCDrc4Fue8JEyY4++7evbtz7d588010794dFRUV6NmzJ77++mttezmuFixYgH79+qGyshJ77LEH7rzzTgghAs8dACxfvhx/+tOfsNtuu6G8vByHHHIInnnmGc96S5YswezZs7O2d8ghh2giGgCUl5fjlFNOwbJly7B582ZneXV1NSoqKjyi1+67756XiNmnTx/0798fAPDzn/8ckUjEcdHIsL6pU6fi2GOPRevWrXHLLbcASItvF154IXbbbTdUVFTgsMMOw3PPPae1LcOz7r//fgwdOhT77rsvWrdujRNPPBFLly6FEAIDBw7EXnvthVatWuGMM87Ahg0bsvZ51apVGDBgAPbaay+Ul5dj9913xxlnnIFFixYBSI+PGTNm4OOPP3bGt3Q+bdiwAX/961/RvXt3tGnTBu3atcPJJ5+Mb7/91ml/woQJ+PnPfw4AGDBggOceseVIq62txfXXX48uXbqgvLwcBx54IO6//37PeIpEIrjyyivxv//9Dz/5yU+c8TNy5Misx+3HL37xCwDA/PnznWWDBw9Gx44d8ec//xlCCNTU1ODaa691RDaTW265BclkEnfffXdefejUqRMOOuggrQ82Fi1ahE6dOgEA7rjjDufcqt+Ts2fPxllnnYWdd94ZFRUV+NnPfoZ33nlHaycej+OOO+7AAQccgIqKCuyyyy445phjMGbMGADZv5tthPkuDfN3J2j85YK8/7777jv07t0brVu3xv7774/XX38dAPDxxx+jV69eaNWqFQ488EBPKPjixYtx+eWX48ADD0SrVq2wyy674Oyzz3buE5Wqqipce+21zvjdf//9cc8993iclytXrsTs2bMRj8cD+z5z5kzMnDkTl1xyiebKvvzyyyGEcI4hDLW1tWhsbLR+NmnSJCQSCZx77rnacvleDQPNNmYIIYTkBh1phBDSQpCTsF122cVZlkgk0K9fPxxzzDG4//77HTfMpZdeimeffRYDBgzA1VdfjYULF+LRRx/F119/jc8++8xxjdx6660YNGgQTjnlFJxyyimYNm0aTjzxRN8Hc5UxY8bgtNNOw+67745rrrkGnTt3xqxZs/Dee+/hmmuuwaWXXooVK1ZgzJgxeOGFF7K2N2PGDPziF79Au3btcOONN6K0tBSPP/44+vTp40yKVK666irstNNOuO2227Bo0SIMGTIEV155JV599VUAaTHjxBNPRKdOnXDTTTehQ4cOWLRoEd58883Afrzwwgt44oknMHnyZDz11FMAgKOOOgpA2hn43HPP4ayzzsL111+PL7/8EoMHD8asWbPw1ltvae3MmTMHv/vd73DppZfi4osvxoEHHmjdX2NjI/r164eGhgZcddVV6Ny5M5YvX4733nsPVVVVaN++PQBg2LBhOOSQQ/CrX/0KJSUlePfdd3H55ZcjlUrhiiuu0NqcN28efv/73+PSSy/Feeedh/vvvx+nn346hg8fjltuuQW
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106 0.53\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNIAAAORCAYAAAA3ZI+fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd5wU5f0H8M/udTpKVVFRY40lIRF7JWKNmhCjxogkGqLRRPklRo2xR2yxY1dQYyexK0UEC4hIlyq9c3DA3XF1b3ee3x97M/s8zzwzO7u3e7feft6+fHE3NzvzzOzs7Mx3vs/3CQkhBIiIiIiIiIiIiMhXuK0bQERERERERERE9H3AQBoREREREREREVEADKQREREREREREREFwEAaERERERERERFRAAykERERERERERERBcBAGhERERERERERUQAMpBEREREREREREQXAQBoREREREREREVEADKQREREREREREREFwEAaEVE7t/fee+Oyyy5zfp8yZQpCoRCmTJmSsXWEQiHcdtttGVtept1///3YZ599UFBQgCOOOKKtm5OTcv09bG01NTW4/PLL0adPH4RCIVx77bVt3aSctXr1aoRCIYwZM6atm0JERESUdQykERFl0ZgxYxAKhZz/S0tLsf/+++Pqq69GeXl5WzcvJR999NH3MtAyYcIEXH/99Tj22GMxevRo3H333W3dpLz25JNP4le/+hX23HNPhEIhJcirq6ysxB/+8Af07NkTHTt2xMknn4zZs2cb5925cyeuv/569O/fHyUlJdh9990xZMgQ1NXVpdXOu+++G2PGjMGVV16Jl19+Gb/97W/TWk6uWLRoEW677TasXr067WW8+uqrePjhhzPWpky47LLLlHNsSUkJ9t9/f9xyyy1oaGgAALz++usIhUJ4+umnjcu48sorUVRUhHnz5gGIP3wIhUK45pprXPPaDyLGjh3rTNPP84WFhdh9991x2WWXYcOGDYG2ozXOr3V1dbjtttsy+hAlqEwcf+1BKuc0nX6s2/8feOCBynwbN27EJZdcggMOOACdO3dGt27dcOSRR+LFF1+EECIbm0VElHcK27oBRET54I477kD//v3R0NCAL7/8Ek8++SQ++ugjLFiwAB06dGjVtpxwwgmor69HcXFxSq/76KOPMGrUKOPNXn19PQoLc/Mr5dNPP0U4HMbzzz+f8jZT5t17773YuXMnjjzySGzatMlzPsuycNZZZ2HevHn429/+hh49euCJJ57ASSedhFmzZuEHP/iBM29VVRVOPPFErF+/Hn/4wx+w3377YevWrfjiiy/Q2NiY1mfs008/xVFHHYVbb701re3MNYsWLcLtt9+Ok046CXvvvXday3j11VexYMECV3beXnvthfr6ehQVFbW8oWkoKSnBc889ByB+LLz77ru48847sWLFCrzyyiu48MIL8eKLL+KGG27Aeeedh969ezuvnTFjBp555hn83//9Hw4//HBluc8++yxuvPFG7LbbboHaIZ/np0+fjjFjxuDLL7/EggULUFpa6vtav/NrptTV1eH2228HAJx00klZW49JJo6/77tUzmle5GPd1rVrV+X3iooKrF+/HkOGDMGee+6JpqYmTJw4EZdddhmWLl3Kh0lERBmQm3c9RETtzBlnnIGf/OQnAIDLL78cu+66Kx588EG8++67uOiii4yvqa2tRceOHTPelnA4nPSmLlWZXl4mbdmyBWVlZRkLogkh0NDQgLKysowsL9989tlnTjZap06dPOcbO3Yspk2bhrfeegtDhgwBAFxwwQXYf//9ceutt+LVV1915r3xxhuxZs0azJ49G/3793em//3vf0+7nVu2bMHBBx+cdL6GhgYUFxcjHM7fJH8727atFBYW4pJLLnF+v+qqq3DMMcfgtddew4MPPojevXvjySefxCGHHILrrrvOOXZisRiGDx+OPffc0xXAOuSQQ7B06VLcc889ePTRRwO1Qz/P9+jRA/feey/ee+89XHDBBZnZWPreSuWc5kU/1k0OO+wwV9bh1VdfjXPOOQePPvoo7rzzThQUFKS9HURExK6dRERt4pRTTgEArFq1CkC8y0anTp2wYsUKnHnmmejcuTN+85vfAIg/xX744YdxyCGHoLS0FL1798bw4cOxY8cOZZlCCNx1113YY4890KFDB5x88slYuHCha91eNdK+/vprnHnmmejevTs6duyIww47DI888ojTvlGjRgGA0qXEZqqvNWfOHJxxxhno0qULOnXqhFNPPRXTp09X5rG7RE2dOhUjRoxwurucf/752Lp1qzLvzJkzMXjwYPTo0QNlZWXo378/fve73/nu51AohNGjR6O2ttZps13HKRqN4s4778S+++6LkpIS7L333rjpppvQ2NioLGPvvffG2WefjfHjx+MnP/kJysrKPLuIAcCyZcvwy1/+En369EFpaSn22GMPXHjhhaiqqnLmGT16NE455RT06tULJSUlOPjgg/Hkk0+6lmWve8qUKc66Dz30UOe9+9///odDDz0UpaWlGDBgAObMmaO83j6uVq5cicGDB6Njx47YbbfdcMcddwTq4rNhwwb87ne/Q+/evVFSUoJDDjkEL7zwgmu+tWvXYsmSJUmXB8Szl+Rjx8vYsWPRu3dv/OIXv3Cm9ezZExdccAHeffdd532qrKzE6NGj8Yc//AH9+/dHJBJxvYepsD8fq1atwocffugcN6tXr3b+9vrrr+Pmm2/G7rvvjg4dOqC6uhoA8NZbb2HAgAEoKytDjx49cMkll7i69tnvydq1a3H22WejU6dO2H333Z3P17fffotTTjkFHTt2xF577RXo5hqId18cMGAAOnfujC5duuDQQw91Pr9jxozBr371KwDAySef7GyTfRy9++67OOuss7DbbruhpKQE++67L+68807EYjFn+SeddBI+/PBDrFmzxnm9nVnkVSPt008/xfHHH4+OHTuiW7duOPfcc7F48WJlnttuuw2hUAjLly/HZZddhm7duqFr164YNmxY2t1yQ6EQjjvuOAghsHLlSgDxz9Jtt92G1157DRMnTgQAPProo5g7dy6efPJJV9bi3nvvjUsvvRTPPvssNm7cmFY7jj/+eADAihUrfOdLdn4N+h3gd45cvXo1evbsCQC4/fbbnXX4ZcA1NTXh9ttvxw9+8AOUlpZi1113xXHHHefsP9uSJUswZMgQ7LLLLigtLcVPfvITvPfee87fkx1/QdmfvzfffBO33347dt99d3Tu3BlDhgxBVVUVGhsbce2116JXr17o1KkThg0b5joXBD33AsDHH3/sHL+dO3fGWWed5fpObWpqwpIlS3yza21Bz2nJxGIx55yTir333ht1dXWIRCLOtCDfV0RE5MaMNCKiNmDfWO26667OtGg0isGDB+O4447DAw884NzYDR8+HGPGjMGwYcPw5z//GatWrcLjjz+OOXPmYOrUqU53qltuuQV33XUXzjzzTJx55pmYPXs2TjvtNOWi2cvEiRNx9tlno2/fvvjLX/6CPn36YPHixfjggw/wl7/8BcOHD8fGjRsxceJEvPzyy0mXt3DhQhx//PHo0qULrr/+ehQVFeHpp5/GSSedhM8++wwDBw5U5r/mmmvQvXt33HrrrVi9ejUefvhhXH311XjjjTcAxLODTjvtNPTs2RM33HADunXrhtWrV+N///ufbztefvllPPPMM5gxY4bTHeaYY44BEM8YefHFFzFkyBD83//9H77++muMHDkSixcvxttvv60sZ+nSpbjoooswfPhwXHHFFTjggAOM64tEIhg8eDAaGxtxzTXXoE+fPtiwYQM++OADVFZWOl1w7OyYn//85ygsLMT777+Pq666CpZl4U9/+pOyzOXLl+Piiy/G8OHDcckll+CBBx7AOeecg6eeego33XQTrrrqKgDAyJEjccEFF2Dp0qVKdlQsFsPpp5+Oo446Cvfddx/GjRuHW2+9FdFoFHfccYfnvisvL8dRRx2FUCiEq6++Gj179sTHH3+M3//+96iurla691166aX47LPPMlp/Z86cOfjxj3/syvQ68sgj8cwzz+C
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24 0.39\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNIAAAORCAYAAAA3ZI+fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd3gU1foH8O/upjdqaIKAiKCIckXFjgXBglfvz4Zer4C9YLlYrthQ1ItdrCCoiO3ae6GIgIUmRRGQ3ktCCIT0bJvfH7tn9syZM7uTZEOR7+d5fEy2zM7MbjbsN+/7Ho9hGAaIiIiIiIiIiIgoLu+e3gEiIiIiIiIiIqJ9AYM0IiIiIiIiIiIiFxikERERERERERERucAgjYiIiIiIiIiIyAUGaURERERERERERC4wSCMiIiIiIiIiInKBQRoREREREREREZELDNKIiIiIiIiIiIhcYJBGRERERERERETkAoM0IqL9UIcOHTBo0CDz++nTp8Pj8WD69OlJewyPx4OHHnooadtLtqeeegoHHXQQfD4fevTosad3Z6+0tz+HyTJx4kT06NEDGRkZ8Hg8KCkp2dO7tNcaNGgQOnTosKd3g4iIiGiPYZBGRLSbvfnmm/B4POZ/GRkZOOSQQzBkyBAUFhbu6d2rlW+//XafDFomT56Mu+++GyeeeCLGjx+P//73v3t6l/ZbGzduxMMPP4xjjz0WTZo0QfPmzXHqqafi+++/T3jfa6+9Fh6PB/3796/z4xcXF+OSSy5BZmYmXn75Zbz99tvIzs6u8/b2Bq+88grefPPNOt9/y5YteOihh/Dbb78lbZ/qa926dZb3Ta/Xi6ZNm+Lss8/GrFmzAACBQADdu3dHp06dUFVVpd1GVlYWLr74YgCx9+KMjAxs3rzZdvtTTz0Vhx9+uOWyDh06WPYjOzsbxx57LN566y1Xx1FZWYmHHnooqX+00NmT7831ff39VXz55Zc46qijkJGRgQMPPBDDhw9HMBhMeL8tW7bgiiuuQJcuXZCbm4vGjRvj2GOPxYQJE2AYhu3277//vvk4+fn5uPrqq7F9+/aGOCQiIopK2dM7QES0vxoxYgQ6duyI6upq/Pzzzxg9ejS+/fZbLF68GFlZWbt1X0455RRUVVUhLS2tVvf79ttv8fLLL2s/sFVVVSElZe/8NfPDDz/A6/Xi9ddfr/UxU3J98cUXeOKJJ3DBBRdg4MCBCAaDeOutt3DmmWfijTfewODBg7X3mzdvHt58801kZGTU6/F//fVXlJWV4ZFHHkGfPn3qta29xSuvvILmzZtbqk5rY8uWLXj44YfRoUMHW7XmuHHjEA6H67+TdXTZZZfhnHPOQSgUwooVK/DKK6/gtNNOw6+//oru3btj7NixOPHEE/HII4/YAvIhQ4YgLS0NL7zwguXympoaPP7443jxxRdd7UOPHj1wxx13AAC2bt2K1157DQMHDkRNTQ2uvfbauPetrKzEww8/DCAS1DWUeO/NDa2+r7+/gu+++w4XXHABTj31VLz44ov4448/8Oijj2Lbtm0YPXp03Ptu374dmzZtwkUXXYQDDzwQgUAAU6ZMwaBBg7B8+XLL63r06NG46aabcMYZZ+DZZ5/Fpk2b8Pzzz2PevHmYM2dOvd8fiYhIb+/8hENEtB84++yzcfTRRwMArrnmGjRr1gzPPvssvvjiC1x22WXa+1RUVDRItYzX6036P7j35n/Ab9u2DZmZmUkL0QzDQHV1NTIzM5Oyvf3Jaaedhg0bNqB58+bmZTfccAN69OiBBx98UBukGYaBW2+9FVdeeSWmTp1ar8fftm0bAKBx48YJb1tZWbnbQ+69TWpq6h59/KOOOgpXXHGF+f3JJ5+Ms88+G6NHj8Yrr7yC448/HjfccAOefvpp/POf/0S3bt0AAJ988gm++eYbvPLKK2jdurVlmz169MC4ceMwbNgwtGnTJuE+HHDAAZZ9GDRoEA466CA899xzCYM02j/ceeedOOKIIzB58mTzD0p5eXn473//i9tuuw1du3Z1vO8RRxxhq1gcMmQIzjvvPLzwwgt45JFH4PP54Pf7ce+99+KUU07BlClT4PF4AAAnnHACzjvvPIwbNw633HJLgx0jEdH+jK2dRER7idNPPx0AsHbtWgCRD2c5OTlYvXo1zjnnHOTm5uKf//wnACAcDmPUqFHo1q0bMjIy0LJlS1x//fXYuXOnZZuGYeDRRx9F27ZtkZWVhdNOOw1LliyxPbbTjLQ5c+bgnHPOQZMmTZCdnY0jjjgCzz//vLl/L7/8MgBYWp0E3XythQsX4uyzz0ZeXh5ycnJwxhlnYPbs2ZbbiHarX375BUOHDkV+fj6ys7Pxj3/8A0VFRZbbzps3D/369UPz5s2RmZmJjh074qqrrop7nj0eD8aPH4+Kigpzn0UbUjAYxCOPPIJOnTohPT0dHTp0wL333ouamhrLNjp06ID+/ftj0qRJOProo5GZmYlXX33V8TFXrlyJCy+8EK1atUJGRgbatm2LAQMGYNeuXeZtxo8fj9NPPx0tWrRAeno6DjvsMG3lgnjs6dOnm4/dvXt387n79NNP0b17d2RkZKBnz55YuHCh5f7idbVmzRr069cP2dnZaNOmDUaMGKFtG1Jt3rwZV111FVq2bIn09HR069YNb7zxhu12GzZswLJlyxJur1u3bpYQDQDS09NxzjnnYNOmTSgrK7Pd5+2338bixYvx2GOPJdx+PKeeeioGDhwIADjmmGPg8XjMKhrR1jd//nyccsopyMrKwr333gsgEr5dffXVaNmyJTIyMnDkkUdiwoQJlm2LVsSnn34aL7/8Mg466CBkZWWhb9++2LhxIwzDwCOPPIK2bdsiMzMT559/Pnbs2JFwnwsKCjB48GC0bdsW6enpaN26Nc4//3ysW7cOQOT1sWTJEsyYMcN8fYvKpx07duDOO+9E9+7dkZOTg7y8PJx99tn4/fffze1Pnz4dxxxzDABg8ODBtp8R3Yy0iooK3HHHHWjXrh3S09PRpUsXPP3007bXk8fjwZAhQ/D555/j8MMPN18/EydOTHjcTk4++WQAwOrVq83LRo4ciebNm+OGG26AYRgoLy/H7bffboZsqnvvvRehUAiPP/54nfYhPz8fXbt2teyDzrp165Cfnw8AePjhh81zK79PLlu2DBdddBGaNm2KjIwMHH300fjyyy8t2wkEAnj44YfRuXNnZGRkoFmzZjjppJMwZcoUAInfm3XcvJe6+b0T7/VXG+Lnb9GiRejduzeysrJw8MEH4+OPPwYAzJgxA7169UJmZia6dOliawVfv349brrpJnTp0gWZmZlo1qwZLr74YvPnRFZSUoLbb7/dfP0efPDBeOKJJ2yVl1u3bsWyZcsQCATi7vvSpUuxdOlSXHfddZaq7JtuugmGYZjHUFsdOnRAZWUl/H4/AGDx4sUoKSnBpZdeanl++/fvj5ycHLz//vuW+7/44ovo1q0bsrKy0KRJExx99NF477336rQvRET7O1akERHtJcSHsGbNmpmXBYNB9OvXDyeddBKefvppsxrm+uuvx5tvvonBgwfj1ltvxdq1a/HSSy9h4cKF+OWXX8yqkQcffBCPPvoozjnnHJxzzjlYsGAB+vbta/5DPJ4pU6agf//+aN26NW677Ta0atUKf/75J77++mvcdtttuP7667FlyxZMmTIFb7/9dsLtLVmyBCeffDLy8vJw9913IzU1Fa+++ipOPfVU80OR7JZbbkGTJk0wfPhwrFu3DqNGjcKQIUPwwQcfAIiEGX379kV+fj7uueceNG7cGOvWrcOnn34adz/efvttjB07FnPnzsVrr70GIPIXfCBSGThhwgRcdNFFuOOOOzBnzhyMHDkSf/75Jz777DPLdpYvX47LLrsM119/Pa699lp06dJF+3h+vx/9+vVDTU0NbrnlFrRq1QqbN2/G119/jZKSEjRq1AhApEWnW7du+Pvf/46UlBR89dVXuOmmmxAOh3HzzTdbtrlq1SpcfvnluP7663HFFVfg6aefxnnnnYcxY8bg3nvvxU033QQgEihccsklWL58Obze2N/
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 0.23\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNIAAAORCAYAAAA3ZI+fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd5hU5f028PtM3770LiBGEUWIGLFX4tpLokYTI2KJJZpEEhNNjNheidEYjQ1LBGM0tsQSC4go/qIiFmygKCIddmGB3WXb1PP+MfOcec5zzpk5Mzuzhb0/18UFOzvlTNmBufkWTdd1HURERERERERERJSRp6sPgIiIiIiIiIiIqCdgkEZEREREREREROQCgzQiIiIiIiIiIiIXGKQRERERERERERG5wCCNiIiIiIiIiIjIBQZpRERERERERERELjBIIyIiIiIiIiIicoFBGhERERERERERkQsM0oiIiIiIiIiIiFxgkEZEtJMbNWoUzjvvPOPrhQsXQtM0LFy4sGC3oWkarr/++oJdX6Hddttt2HXXXeH1ejFx4sSuPpxuqbs/h52tubkZF154IQYPHgxN0/CrX/2qqw+p21q9ejU0TcOcOXO6+lCIiIiIio5BGhFREc2ZMweaphm/QqEQdt99d1x++eWoq6vr6sPLySuvvNIjg5bXXnsNv/3tb3HwwQdj9uzZuOWWW7r6kHq1+++/H2eccQZ22WUXaJpmCnlVDQ0N+NnPfoYBAwagrKwMRx55JJYsWWJ73hdffBH77rsvQqEQdtllF8yYMQOxWCzv47zlllswZ84cXHrppXjsscfw05/+NO/r6g6++OILXH/99Vi9enXe1/HEE0/gzjvvLNgxFcJ5551neo8NBoPYfffdcd1116G9vR0A8OSTT0LTNDzwwAO213HppZfC7/fj008/BZD8zwdN03DFFVdYziv+I+LZZ581TlPf530+H4YNG4bzzjsPGzZscHU/OuP9tbW1Fddff31B/xPFrUK8/nYGubynyRKJBObMmYOTTz4ZI0aMQFlZGfbee2/cfPPNxutcaGtrwwUXXIC9994bVVVVKC8vx4QJE3DXXXchGo0W664REfUqvq4+ACKi3uDGG2/E6NGj0d7ejrfffhv3338/XnnlFSxduhSlpaWdeiyHHXYY2traEAgEcrrcK6+8gnvvvdf2w15bWxt8vu75V8obb7wBj8eDv//97znfZyq8W2+9FTt27MD++++PTZs2OZ4vkUjghBNOwKeffoqrrroK/fv3x3333YcjjjgCH330Eb7zne8Y53311Vdx6qmn4ogjjsDdd9+Nzz//HDfffDM2b96M+++/P6/jfOONN3DAAQdgxowZeV2+u/niiy9www034IgjjsCoUaPyuo4nnngCS5cutVTnjRw5Em1tbfD7/R0/0DwEg0E8/PDDAIDGxka88MILuOmmm7By5Uo8/vjjOOuss/Doo4/i6quvxqmnnopBgwYZl33//ffx4IMP4te//jUmTJhgut6HHnoI11xzDYYOHerqOOT3+ffeew9z5szB22+/jaVLlyIUCmW8bKb310JpbW3FDTfcAAA44ogjinY7dgrx+uvpcnlPU7W2tmLatGk44IADcMkll2DgwIFYtGgRZsyYgQULFuCNN96ApmkAkn8fL1u2DMcffzxGjRoFj8eDd999F1deeSUWL16MJ554orPuMhHRTqt7fuohItrJHHfccdhvv/0AABdeeCH69euHO+64Ay+88ALOPvts28u0tLSgrKys4Mfi8XiyfqjLVaGvr5A2b96MkpKSgoVouq6jvb0dJSUlBbm+3uatt94yqtHKy8sdz/fss8/i3XffxTPPPIPTTz8dAHDmmWdi9913x4wZM0wfBn/zm99gn332wWuvvWYEupWVlbjlllvwy1/+EmPHjs35ODdv3oxx48ZlPV97ezsCgQA8nt5b5C+qbbuKz+fDOeecY3x92WWX4aCDDsK//vUv3HHHHRg0aBDuv/9+7LXXXrjyyiuN1048HsfFF1+MXXbZxRJg7bXXXvjqq6/wpz/9CX/7299cHYf6Pt+/f3/ceuutePHFF3HmmWcW5s5Sj5XLe5oqEAjgnXfewUEHHWScdtFFF2HUqFFGmDZlyhQAQN++ffHee++ZLn/JJZegqqoK99xzD+644w4MHjy4CPeQiKj36L3/6iMi6kJHHXUUAGDVqlUAku1J5eXlWLlyJY4//nhUVFTgJz/5CYDk/2Lfeeed2GuvvRAKhTBo0CBcfPHF2L59u+k6dV3HzTffjOHDh6O0tBRHHnkkli1bZrltpxlpixcvxvHHH48+ffqgrKwM++yzD+666y7j+O69914AMLUwCXbztT7++GMcd9xxqKysRHl5OY4++mjLP+5FS9Q777yD6dOnG+0up512GrZs2WI674cffoiamhr0798fJSUlGD16NM4///yMj7OmaZg9ezZaWlqMYxZznGKxGG666SaMGTMGwWAQo0aNwu9//3uEw2HTdYwaNQonnngi5s2bh/322w8lJSWOLWIAsGLFCvzwhz/E4MGDEQqFMHz4cJx11llobGw0zjN79mwcddRRGDhwIILBIMaNG2dbOSVue+HChcZtjx8/3nju/vOf/2D8+PEIhUKYNGkSPv74Y9Plxevq22+/RU1NDcrKyjB06FDceOON0HU942MHABs2bMD555+PQYMGIRgMYq+99sIjjzxiOd/atWuxfPnyrNcHJKuX5NeOk2effRaDBg3CD37wA+O0AQMG4Mwzz8QLL7xgPE9ffPEFvvjiC/zsZz8zVUVedtll0HXd1ILnhvj5WLVqFV5++WXjdbN69Wrje08++SSuvfZaDBs2DKWlpWhqagIAPPPMM5g0aRJKSkrQv39/nHPOOZbWPvGcrF27FieeeCLKy8sxbNgw4+fr888/x1FHHYWysjKMHDnSdfXIk08+iUmTJqGiogKVlZUYP3688fM7Z84cnHHGGQCAI4880rhP4nX0wgsv4IQTTsDQoUMRDAYxZswY3HTTTYjH48b1H3HEEXj55ZexZs0a4/KisshpRtobb7yBQw89FGVlZaiursYpp5yCL7/80nSe66+/Hpqm4ZtvvsF5552H6upqVFVVYdq0aWhtbXV131WapuGQQw6Bruv49ttvASR/lq6//nr861//wvz58wEAf/vb3/DJJ5/g/vvvt1QGjxo1Cueeey4eeughbNy4Ma/jOPTQQwEAK1euzHi+bO+vbv8OyPQeuXr1agwYMAAAcMMNNxi3kakCLhqN4oYbbsB3vvMdhEIh9OvXD4cccojx+AnLly/H6aefjr59+yIUCmG//fbDiy++aHw/2+vPLfHz9/TTT+OGG27AsGHDUFFRgdNPPx2NjY0Ih8P41a9+hYEDB6K8vBzTpk2zvJ+7fe8FkpWu4vVbUVGBE044wfJ3ajQaxfLlyzNW1wpu39PsBAIBU4gmnHbaaQBg+bmyI35eGxoajNNqa2sxbdo0DB8+HMFgEEOGDMEpp5zS61twiYiyYUUaEVEXEB+s+vXrZ5wWi8VQU1ODQw45BLfffrvxwe7iiy/GnDlzMG3aNPziF7/AqlWrcM899+Djjz/GO++8Y7RTXXfddbj55ptx/PHH4/jjj8eSJUtwzDHHIBKJZD2e+fPn48QTT8SQIUPwy1/+EoMHD8aXX36Jl156Cb/85S9x8cUXY+PGjZg/fz4ee+yxrNe3bNkyHHrooaisrMRvf/tb+P1+PPDAAzjiiCPw1ltvYfLkyabzX3HFFejTpw9mzJiB1atX484778Tll1+Op556CkCyOuiYY47BgAEDcPXVV6O6uhqrV6/Gf/7zn4zH8dhjj+HBBx/E+++/b7R+iQ8jF154IR599FGcfvrp+PWvf43Fixdj5syZ+PLLL/Hcc8+Zruerr77C2WefjYsvvhgXXXQR9thjD9vbi0QiqKmpQTgcxhVXXIHBgwdjw4YNeOmll9DQ0ICqqioAMKpjTj75ZPh8Pvz3v//FZZddhkQigZ///Oem6/zmm2/w4x//GBdffDHOOecc3H777TjppJMwa9Ys/P73v8dll10
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "119 0.22\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNIAAAORCAYAAAA3ZI+fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd5xU9fX/8fe0ne1LW5p0UBRESFCxExXFEqOJPfFrLz8VS4glJFHEEnuLDUtUNBp7iQ1FBCuKCiggVXrZBRa215m5vz9m7p17p+3sskOR1/Px4MHunTt37tyZ3WXfnHM+LsMwDAEAAAAAAABIyb29TwAAAAAAAADYGRCkAQAAAAAAAGkgSAMAAAAAAADSQJAGAAAAAAAApIEgDQAAAAAAAEgDQRoAAAAAAACQBoI0AAAAAAAAIA0EaQAAAAAAAEAaCNIAAAAAAACANBCkAcAvXJ8+fXTuuedan0+fPl0ul0vTp09vs8dwuVy66aab2ux4be3uu+9Wv3795PF4NGzYsO19OjukHf013NYCgYCuu+469ezZU263WyeddNL2PqUdGu8fAACwqyBIA4AMevbZZ+Vyuaw/2dnZ2mOPPTRmzBiVlpZu79Nrkffff3+n/EX5o48+0nXXXaeDDz5YzzzzjP75z39u71PapT322GM69dRT1atXL7lcLkfIa7d+/Xr99a9/1eGHH66CgoKU4W9TU5MmTJigfv36ye/3q1+/frr11lsVCARafZ5PP/207r77bp1yyimaNGmS/vznP7f6WDuCdevW6aabbtKcOXNafYwd8XvATTfd5Pge6/P51KdPH1155ZUqLy+XJM2YMUNut1vjxo1LeIw777xTLpdL7733niTpN7/5jVwul0444YS4fVesWCGXy6V77rnH2mb+54T5x+PxqHPnzjrllFO0YMGCtJ7HV199pZtuusk650z55z//qbfeeiujj5FIW7z/fgkaGhp0/fXXq3v37srJydGIESM0ZcqUtO77xhtv6PTTT1e/fv2Um5urgQMH6i9/+Uvce6asrEx33323DjvsMBUXF6tdu3Y64IAD9PLLL2fgGQHArsm7vU8AAHYFN998s/r27av6+np98cUXeuyxx/T+++9r3rx5ys3N3abncthhh6murk5ZWVktut/777+vRx55JOEv0nV1dfJ6d8wfKZ988oncbrf+/e9/t/g5o+3deeedqqqq0v7776/169cn3W/RokW68847tfvuu2vIkCGaMWNG0n3POussvfrqqzr//PO177776uuvv9YNN9ygVatW6YknnmjVeX7yySfabbfddP/997fq/juadevWacKECerTp0+rqzJ35O8Bjz32mPLz81VTU6OpU6fqoYce0qxZs/TFF1/owAMP1CWXXKJ7771XZ511lgYPHmzdb+XKlbr55pt16qmn6vjjj3cc891339X333+v4cOHp3UOV155pfbbbz81NTXpxx9/1MSJEzV9+nTNmzdPXbt2TXnfr776ShMmTNC5556rdu3atfj5p+uf//ynTjnllG1eYdkW779fgnPPPVevvfaarr76au2+++569tlnddxxx2natGk65JBDUt734osvVvfu3XXWWWepV69emjt3rh5++GG9//77mjVrlnJyciSFg+O///3vOu644/SPf/xDXq9Xr7/+us444wz99NNPmjBhwrZ4qgDwi7Zj/tYDAL8wxx57rPbdd19J0oUXXqiOHTvqvvvu09tvv60zzzwz4X1qamqUl5fX5ufidruVnZ3dpsds6+O1pQ0bNignJ6fNQjTDMFRfX2/90oKW+fTTT61qtPz8/KT7DR8+XGVlZerQoYNee+01nXrqqQn3+/bbb/XKK6/ohhtu0M033yxJ+n//7/+pU6dOuu+++zRmzBjts88+LT7PDRs2pBVoBAIBhUKhXT6k3d7fA0455RR16tRJknTJJZfojDPO0Msvv6yZM2dq//331x133KG3335bl1xyiT7//HO5XC5J0hVXXCGfz6cHH3zQcbxevXqpqqpKEyZM0P/+97+0zuHQQw/VKaecYn0+cOBAXXrppXruued03XXXtdEzxc5q5syZeumll3T33XfrmmuukSSdffbZ2nvvvXXdddfpq6++Snn/1157Tb/5zW8c24YPH65zzjlHL7zwgi688EJJ0uDBg7VkyRL17t3b2u+yyy7TqFGjdOedd+q6667LyL8tAGBXQmsnAGwHRxxxhCRp+fLlksL/S52fn6+ff/5Zxx13nAoKCvSnP/1JkhQKhfTAAw9o8ODBys7OVpcuXXTJJZdoy5YtjmMahqFbb71VPXr0UG5urg4//HDNnz8/7rGTzUj75ptvdNxxx6l9+/bKy8vTPvvsY/1yee655+qRRx6RJEcLkynRfKTZs2fr2GOPVWFhofLz83XkkUfq66+/duxjtr5++eWXGjt2rIqLi5WXl6ff//732rhxo2Pf7777TqNHj1anTp2Uk5Ojvn376vzzz095nV0ul5555hnV1NRY5/zss89KCgcgt9xyi/r37y+/368+ffrob3/7mxoaGhzH6NOnj37729/qww8/1L777qucnBw9/vjjSR9zyZIlOvnkk9W1a1dlZ2erR48eOuOMM1RRUWHt88wzz+iII45Q586d5ff7NWjQID322GNxxzIfe/r06dZjDxkyxHrt3njjDQ0ZMkTZ2dkaPny4Zs+e7bi/+b5atmyZRo8erby8PHXv3l0333yzDMNIee0kae3atTr//PPVpUsX+f1+DR48WE8//XTcfqtWrdLChQubPZ4k9e7d2/HeSaagoEAdOnRodr/PP/9cknTGGWc4tp9xxhkyDKPF7Uxm6960adM0f/58630zffp0R1vfAw88YL13fvrpJ0nhKrZDDz1UeXl5ateunU488cS41j6zFXHx4sU666yzVFRUpOLiYt1www0yDEOrV6/WiSeeqMLCQnXt2lX33ntvWuc9ZcoUHXLIIWrXrp3y8/M1cOBA/e1vf5MU/prfb7/9JEnnnXde3NfC559/brXb+v1+9ezZU3/+859VV1dnHX9H+B7QEoceeqgk6eeff5YkFRUV6cEHH9SXX36pp556SpL05ptv6p133tEdd9yhbt26Oe5fUFCgP//5z3rnnXc0a9asNjmHZG666SZde+21kqS+ffta13bFihXWPv/5z380fPhw5eTkqEOHDjrjjDO0evVqx3Ga+97jcrlUU1OjSZMmWY+RrLXa9NBDD2nw4MHKzc1V+/btte++++rFF1907NPc94nm3n/psn/9PfLII1aL49FHH63Vq1fLMAzdcsst6tGjh3JycnTiiSdq8+bNjmO8/fbbOv7449W9e3f5/X71799ft9xyi4LBYNzjffPNNzrmmGNUVFSk3NxcjRw5Ul9++WXcfgsXLtSqVauaPf/XXntNHo9HF198sbUtOztbF1xwgWbMmBH3esaKDdEk6fe//70kOb7P9O3b1xGiSeHX/qSTTlJDQ4OWLVtmba+qqtLVV1+tPn36yO/3q3PnzjrqqKNa/Z4HgF0FFWkAsB2Yv1h17NjR2hYIBDR69Ggdcsghuueee6yWz0suuUTPPvuszjvvPF155ZVavny5Hn74Yc2ePVtffvmlfD6fJOnGG2/UrbfequOOO07HHXecZs2apaOPPlqNjY3Nns+UKVP029/+Vt26ddNVV12lrl27asGCBXr33Xd11VVX6ZJLLtG6des0ZcoUPf/8880eb/78+Tr00ENVWFio6667Tj6fT48//rh+85vf6NNPP9WIESMc+19xxRVq3769xo8frxUrVuiBBx7QmDFjrBBkw4YNOvroo1VcXKy//vWvateunVasWKE33ngj5Xk8//zzeuKJJzRz5kzrl+eDDjpIUrgycNKkSTrllFP0l7/8Rd98841uv/12LViwQG+++abjOIsWLdKZZ56pSy65RBdddJEGDhyY8PEaGxs1evRoNTQ06IorrlDXrl21du1avfvuuyovL1dRUZGkcBva4MGD9bvf/U5er1fvvPOOLrvsMoVCIV1++eWOYy5dulR//OMfdckll+iss87SPffcoxN
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "mean_mae = 0\n",
    "samples = []\n",
    "split = 'test'\n",
    "station = station_with_worst_res_test_set\n",
    "\n",
    "for i in range(3): \n",
    "    \n",
    "    gen = train.get_data_generator(split=split, station=station , sampling_rate=sampling_rate, path=data_path, \n",
    "                                  window='fixed')\n",
    "    station_mae = []\n",
    "    with torch.no_grad():\n",
    "        for i in range(len(gen)): \n",
    "            # idx = np.random.randint(len(gen))\n",
    "            idx = i\n",
    "            sample = gen[idx]\n",
    "            samples.append(sample)\n",
    "            pred = model(torch.tensor(sample[\"X\"], device=model.device).unsqueeze(0))  \n",
    "            pred = pred[0].cpu().numpy()\n",
    "    \n",
    "            pred_pick_idx  = np.argmax(pred[0])\n",
    "            true_pick_idx = np.argmax(sample['y'][0])            \n",
    "                \n",
    "            mae_error = np.abs(pred_pick_idx - true_pick_idx) /100 #mae in seconds\n",
    "            station_mae.append(mae_error)\n",
    "    \n",
    "    sorted = np.argsort(station_mae)[::-1]\n",
    "    mean_mae = np.mean(station_mae)\n",
    "\n",
    "print(np.array(station_mae)[sorted])\n",
    "\n",
    "\n",
    "## plot samples with mae error at leas 0.2s\n",
    "for idx in sorted:\n",
    "    if station_mae[idx] < 0.2: \n",
    "        break\n",
    "    print(idx, station_mae[idx])\n",
    "    plot_sample(samples[idx], model, idx, desc=f\" from station {station} {split} set\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e599f32-b888-4235-8d15-f9d2c98a59cf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}