{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "05600e0a-8b4b-42ab-96d5-9d6eb2c72102",
   "metadata": {},
   "source": [
    "# Check model performance depending on station\n",
    "\n",
    "#### Samples generated with random window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1732f721-6b13-4fb3-8755-dc63cb255285",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mkmilian\u001b[0m (\u001b[33mepos\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /Users/krystynamilian/.netrc\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from obspy.core.event import read_events\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import seisbench.models as sbm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "import seisbench.data as sbd\n",
    "import seisbench.generate as sbg\n",
    "import seisbench.models as sbm\n",
    "from seisbench.util import worker_seeding\n",
    "import numpy as np\n",
    "from torch.utils.data import DataLoader\n",
    "from pathlib import Path\n",
    "import wandb\n",
    "import os\n",
    "import sys\n",
    "\n",
    "from pathlib import Path\n",
    "cwd = str(Path.cwd().parent)\n",
    "sys.path.append(cwd)\n",
    "from scripts import train\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1a322214-ba56-4651-966d-f9afdcfecbc5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "wandb version 0.15.4 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.15.3"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/Users/krystynamilian/Documents/praca/Cyfronet/epos/ai/repo/demo_scripts/notebooks/wandb/run-20230704_110544-8fry08nf</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/epos/demo_scripts-notebooks/runs/8fry08nf' target=\"_blank\">fiery-sky-8</a></strong> to <a href='https://wandb.ai/epos/demo_scripts-notebooks' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/epos/demo_scripts-notebooks' target=\"_blank\">https://wandb.ai/epos/demo_scripts-notebooks</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/epos/demo_scripts-notebooks/runs/8fry08nf' target=\"_blank\">https://wandb.ai/epos/demo_scripts-notebooks/runs/8fry08nf</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m:   1 of 1 files downloaded.  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "PhaseNet(\n",
       "  (inc): Conv1d(3, 8, kernel_size=(7,), stride=(1,), padding=same)\n",
       "  (in_bn): BatchNorm1d(8, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (down_branch): ModuleList(\n",
       "    (0): ModuleList(\n",
       "      (0): Conv1d(8, 8, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (1): BatchNorm1d(8, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(8, 8, kernel_size=(7,), stride=(4,), padding=(3,), bias=False)\n",
       "      (3): BatchNorm1d(8, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (1): ModuleList(\n",
       "      (0): Conv1d(8, 16, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (1): BatchNorm1d(16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(16, 16, kernel_size=(7,), stride=(4,), bias=False)\n",
       "      (3): BatchNorm1d(16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (2): ModuleList(\n",
       "      (0): Conv1d(16, 32, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (1): BatchNorm1d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(32, 32, kernel_size=(7,), stride=(4,), bias=False)\n",
       "      (3): BatchNorm1d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (3): ModuleList(\n",
       "      (0): Conv1d(32, 64, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (1): BatchNorm1d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(64, 64, kernel_size=(7,), stride=(4,), bias=False)\n",
       "      (3): BatchNorm1d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (4): ModuleList(\n",
       "      (0): Conv1d(64, 128, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (1): BatchNorm1d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2-3): 2 x None\n",
       "    )\n",
       "  )\n",
       "  (up_branch): ModuleList(\n",
       "    (0): ModuleList(\n",
       "      (0): ConvTranspose1d(128, 64, kernel_size=(7,), stride=(4,), bias=False)\n",
       "      (1): BatchNorm1d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(128, 64, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (3): BatchNorm1d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (1): ModuleList(\n",
       "      (0): ConvTranspose1d(64, 32, kernel_size=(7,), stride=(4,), bias=False)\n",
       "      (1): BatchNorm1d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(64, 32, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (3): BatchNorm1d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (2): ModuleList(\n",
       "      (0): ConvTranspose1d(32, 16, kernel_size=(7,), stride=(4,), bias=False)\n",
       "      (1): BatchNorm1d(16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(32, 16, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (3): BatchNorm1d(16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (3): ModuleList(\n",
       "      (0): ConvTranspose1d(16, 8, kernel_size=(7,), stride=(4,), bias=False)\n",
       "      (1): BatchNorm1d(8, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): Conv1d(16, 8, kernel_size=(7,), stride=(1,), padding=same, bias=False)\n",
       "      (3): BatchNorm1d(8, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (out): Conv1d(8, 2, kernel_size=(1,), stride=(1,), padding=same)\n",
       "  (softmax): Softmax(dim=1)\n",
       ")"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = train.load_model()\n",
    "\n",
    "run = wandb.init()\n",
    "artifact = run.use_artifact('epos/training_seisbench_models_on_igf_data/model:v113', type='model')\n",
    "artifact_dir = artifact.download()\n",
    "fname = artifact_dir + \"/\" + os.listdir(artifact_dir)[0]\n",
    "\n",
    "model.load_state_dict(torch.load(fname))\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "75cadd4b-7a0e-44cd-acc6-fef26f2cf551",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../../../data/igf/seisbench_format'\n",
    "sampling_rate = 100\n",
    "data = sbd.WaveformDataset(data_path, sampling_rate=sampling_rate)\n",
    "data.filter(data.metadata.trace_Pg_arrival_sample.notna())\n",
    "\n",
    "pick_mae = train.PickMAE(sampling_rate)\n",
    "splits = ['train', 'dev', 'test']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9cdb971d-a08e-49d8-97be-b6aaa53b083a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All samples: 18002\n",
      "Training examples: 12444 69.1%\n",
      "Development examples: 2773 15.4%\n",
      "Test examples: 2785 15.5 %\n"
     ]
    }
   ],
   "source": [
    "all_samples = len(data.train()) + len(data.dev()) + len(data.test())\n",
    "print(f\"All samples: {all_samples}\")\n",
    "print(f\"Training examples: {len(data.train())} {len(data.train())/all_samples * 100:.1f}%\" )\n",
    "print(f\"Development examples: {len(data.dev())} {len(data.dev())/all_samples * 100:.1f}%\")\n",
    "print(f\"Test examples: {len(data.test())} {len(data.test())/all_samples * 100:.1f} %\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccf454fe-e55a-47bc-882a-3e39395147a5",
   "metadata": {},
   "source": [
    "### Calculate overall model performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4c7b4137-4599-4599-9e3e-064145bfccb4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Model resutls for train set\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025075\n",
      "Test avg mae: 0.046459\n",
      "\n",
      "\n",
      "\n",
      "Model resutls for dev set\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025158\n",
      "Test avg mae: 0.049454\n",
      "\n",
      "\n",
      "\n",
      "Model resutls for test set\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025469\n",
      "Test avg mae: 0.047190\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for split in splits: \n",
    "    print(f\"\\n\\nModel resutls for {split} set\")\n",
    "\n",
    "    gen = train.get_data_generator(split=split, station=None, sampling_rate=sampling_rate, path=data_path, window='random')\n",
    "\n",
    "    data_loader = DataLoader(gen, batch_size=256, shuffle=False, num_workers=0,\n",
    "                                    worker_init_fn=worker_seeding)\n",
    "        \n",
    "    test_loss, test_mae = train.test_one_epoch(model, data_loader, pick_mae, wandb_log=False)\n",
    "\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a361bc6a-2dac-4e27-9417-3abe482161a5",
   "metadata": {},
   "source": [
    "## Check # frames per station in each set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b23b71b7-ba71-4e86-8978-5a70d4fbbdfb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: title={'center': 'Frames per station'}, xlabel='station_code'>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 46184 characters]",
      "text/plain": [
       "<Figure size 1500x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "frames_per_station = []\n",
    "for split in splits: \n",
    "    frames_per_station.append(data.get_split(split).metadata.groupby('station_code').count()['index'])\n",
    "    \n",
    "frames_per_station = pd.DataFrame(frames_per_station, index=splits).transpose()\n",
    "frames_per_station.plot(kind='bar', figsize=(15,3), title='Frames per station')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acd81c2b-318f-45ea-b639-bf1ed26b0c10",
   "metadata": {},
   "source": [
    "## Calculate MAE per station for train/dev/test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "66992bf3-c49a-4885-8ab1-5b372ad65202",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train\n",
      "BRDW\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.029197\n",
      "Test avg mae: 0.101062\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028921\n",
      "Test avg mae: 0.080500\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.029244\n",
      "Test avg mae: 0.086313\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.029136\n",
      "Test avg mae: 0.099563\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.029101\n",
      "Test avg mae: 0.100688\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028700\n",
      "Test avg mae: 0.079062\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.029309\n",
      "Test avg mae: 0.098625\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.029184\n",
      "Test avg mae: 0.084937\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.029323\n",
      "Test avg mae: 0.086125\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.029393\n",
      "Test avg mae: 0.103312\n",
      "\n",
      "GROD\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025653\n",
      "Test avg mae: 0.062547\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025550\n",
      "Test avg mae: 0.056665\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025562\n",
      "Test avg mae: 0.055145\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025711\n",
      "Test avg mae: 0.059011\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025634\n",
      "Test avg mae: 0.057455\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025523\n",
      "Test avg mae: 0.054818\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025585\n",
      "Test avg mae: 0.056393\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025612\n",
      "Test avg mae: 0.055862\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025611\n",
      "Test avg mae: 0.058877\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025614\n",
      "Test avg mae: 0.058422\n",
      "\n",
      "GUZI\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024391\n",
      "Test avg mae: 0.035747\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024418\n",
      "Test avg mae: 0.036143\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024409\n",
      "Test avg mae: 0.035239\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024416\n",
      "Test avg mae: 0.035553\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024445\n",
      "Test avg mae: 0.035810\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024413\n",
      "Test avg mae: 0.037195\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024443\n",
      "Test avg mae: 0.035182\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024415\n",
      "Test avg mae: 0.036012\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024411\n",
      "Test avg mae: 0.036120\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024401\n",
      "Test avg mae: 0.036362\n",
      "\n",
      "JEDR\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024772\n",
      "Test avg mae: 0.036505\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024748\n",
      "Test avg mae: 0.034873\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024829\n",
      "Test avg mae: 0.036079\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024805\n",
      "Test avg mae: 0.035852\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024755\n",
      "Test avg mae: 0.034627\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024716\n",
      "Test avg mae: 0.035023\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024677\n",
      "Test avg mae: 0.033938\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024704\n",
      "Test avg mae: 0.033226\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024675\n",
      "Test avg mae: 0.034878\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024956\n",
      "Test avg mae: 0.038911\n",
      "\n",
      "MOSK2\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024905\n",
      "Test avg mae: 0.041251\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024831\n",
      "Test avg mae: 0.040558\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024893\n",
      "Test avg mae: 0.041713\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024854\n",
      "Test avg mae: 0.039448\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024832\n",
      "Test avg mae: 0.040370\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024837\n",
      "Test avg mae: 0.040740\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024842\n",
      "Test avg mae: 0.040198\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024900\n",
      "Test avg mae: 0.040395\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024862\n",
      "Test avg mae: 0.039904\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024800\n",
      "Test avg mae: 0.040857\n",
      "\n",
      "NWLU\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024434\n",
      "Test avg mae: 0.034428\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024436\n",
      "Test avg mae: 0.034107\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024587\n",
      "Test avg mae: 0.034464\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024417\n",
      "Test avg mae: 0.034171\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024523\n",
      "Test avg mae: 0.033232\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024620\n",
      "Test avg mae: 0.034845\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024388\n",
      "Test avg mae: 0.033914\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024565\n",
      "Test avg mae: 0.034201\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024580\n",
      "Test avg mae: 0.034112\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024502\n",
      "Test avg mae: 0.033727\n",
      "\n",
      "PCHB\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024624\n",
      "Test avg mae: 0.041649\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024515\n",
      "Test avg mae: 0.039478\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024490\n",
      "Test avg mae: 0.040745\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024513\n",
      "Test avg mae: 0.039800\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024597\n",
      "Test avg mae: 0.042827\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024525\n",
      "Test avg mae: 0.039057\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024588\n",
      "Test avg mae: 0.042966\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024519\n",
      "Test avg mae: 0.039862\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024510\n",
      "Test avg mae: 0.040924\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024571\n",
      "Test avg mae: 0.040463\n",
      "\n",
      "PPOL\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025924\n",
      "Test avg mae: 0.061875\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026050\n",
      "Test avg mae: 0.066563\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025983\n",
      "Test avg mae: 0.067147\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025980\n",
      "Test avg mae: 0.065153\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026014\n",
      "Test avg mae: 0.061698\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026035\n",
      "Test avg mae: 0.061963\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025978\n",
      "Test avg mae: 0.063233\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025947\n",
      "Test avg mae: 0.063183\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025983\n",
      "Test avg mae: 0.065201\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026024\n",
      "Test avg mae: 0.064024\n",
      "\n",
      "RUDN\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025199\n",
      "Test avg mae: 0.044886\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025190\n",
      "Test avg mae: 0.045990\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025130\n",
      "Test avg mae: 0.045036\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025101\n",
      "Test avg mae: 0.043167\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025226\n",
      "Test avg mae: 0.047723\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025385\n",
      "Test avg mae: 0.061125\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025306\n",
      "Test avg mae: 0.046457\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025231\n",
      "Test avg mae: 0.047450\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025327\n",
      "Test avg mae: 0.047645\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025388\n",
      "Test avg mae: 0.047236\n",
      "\n",
      "RYNR\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025864\n",
      "Test avg mae: 0.066633\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025840\n",
      "Test avg mae: 0.065376\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025626\n",
      "Test avg mae: 0.062124\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025880\n",
      "Test avg mae: 0.066312\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025988\n",
      "Test avg mae: 0.067691\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025817\n",
      "Test avg mae: 0.067727\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025806\n",
      "Test avg mae: 0.065661\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025949\n",
      "Test avg mae: 0.063175\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025792\n",
      "Test avg mae: 0.066198\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025799\n",
      "Test avg mae: 0.066651\n",
      "\n",
      "RZEC\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023818\n",
      "Test avg mae: 0.031034\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023787\n",
      "Test avg mae: 0.028276\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023800\n",
      "Test avg mae: 0.024483\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023806\n",
      "Test avg mae: 0.025862\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023858\n",
      "Test avg mae: 0.028966\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023822\n",
      "Test avg mae: 0.024828\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023817\n",
      "Test avg mae: 0.027586\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023799\n",
      "Test avg mae: 0.031379\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023909\n",
      "Test avg mae: 0.031724\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023884\n",
      "Test avg mae: 0.030345\n",
      "\n",
      "SGOR\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024467\n",
      "Test avg mae: 0.037522\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024469\n",
      "Test avg mae: 0.036812\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024430\n",
      "Test avg mae: 0.035673\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024438\n",
      "Test avg mae: 0.034225\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024428\n",
      "Test avg mae: 0.034473\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024397\n",
      "Test avg mae: 0.036998\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024478\n",
      "Test avg mae: 0.036346\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024488\n",
      "Test avg mae: 0.035342\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024540\n",
      "Test avg mae: 0.038125\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024425\n",
      "Test avg mae: 0.035026\n",
      "\n",
      "TRBC2\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024715\n",
      "Test avg mae: 0.044793\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024758\n",
      "Test avg mae: 0.046050\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024701\n",
      "Test avg mae: 0.042903\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024757\n",
      "Test avg mae: 0.045986\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024681\n",
      "Test avg mae: 0.042041\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024607\n",
      "Test avg mae: 0.044536\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024924\n",
      "Test avg mae: 0.045319\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024695\n",
      "Test avg mae: 0.044265\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024710\n",
      "Test avg mae: 0.045154\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024677\n",
      "Test avg mae: 0.045740\n",
      "\n",
      "TRN2\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025530\n",
      "Test avg mae: 0.054768\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025488\n",
      "Test avg mae: 0.050913\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025473\n",
      "Test avg mae: 0.054110\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025458\n",
      "Test avg mae: 0.057963\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025466\n",
      "Test avg mae: 0.054473\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025534\n",
      "Test avg mae: 0.054216\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025420\n",
      "Test avg mae: 0.049408\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025467\n",
      "Test avg mae: 0.053733\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025430\n",
      "Test avg mae: 0.051889\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025404\n",
      "Test avg mae: 0.054043\n",
      "\n",
      "TRZS\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024975\n",
      "Test avg mae: 0.044880\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025067\n",
      "Test avg mae: 0.043206\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025030\n",
      "Test avg mae: 0.043493\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024915\n",
      "Test avg mae: 0.044593\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025043\n",
      "Test avg mae: 0.044641\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025157\n",
      "Test avg mae: 0.045072\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024961\n",
      "Test avg mae: 0.042823\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025094\n",
      "Test avg mae: 0.044450\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025002\n",
      "Test avg mae: 0.043589\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025029\n",
      "Test avg mae: 0.044689\n",
      "\n",
      "ZMST\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025080\n",
      "Test avg mae: 0.049731\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025050\n",
      "Test avg mae: 0.048796\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025179\n",
      "Test avg mae: 0.048982\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025070\n",
      "Test avg mae: 0.049723\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025061\n",
      "Test avg mae: 0.048439\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025102\n",
      "Test avg mae: 0.049314\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025127\n",
      "Test avg mae: 0.050508\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025169\n",
      "Test avg mae: 0.049843\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025097\n",
      "Test avg mae: 0.051237\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025070\n",
      "Test avg mae: 0.048802\n",
      "\n",
      "LUBW\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028350\n",
      "Test avg mae: 0.090000\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027360\n",
      "Test avg mae: 0.064848\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027597\n",
      "Test avg mae: 0.074545\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028823\n",
      "Test avg mae: 0.088182\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028706\n",
      "Test avg mae: 0.110000\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028709\n",
      "Test avg mae: 0.093333\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028397\n",
      "Test avg mae: 0.078182\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028619\n",
      "Test avg mae: 0.095455\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028573\n",
      "Test avg mae: 0.093333\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028804\n",
      "Test avg mae: 0.119091\n",
      "\n",
      "DWOL\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024134\n",
      "Test avg mae: 0.027203\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024156\n",
      "Test avg mae: 0.027302\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024178\n",
      "Test avg mae: 0.027137\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024149\n",
      "Test avg mae: 0.027665\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024316\n",
      "Test avg mae: 0.036156\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024161\n",
      "Test avg mae: 0.027512\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024177\n",
      "Test avg mae: 0.035867\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024151\n",
      "Test avg mae: 0.027882\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024269\n",
      "Test avg mae: 0.035224\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024150\n",
      "Test avg mae: 0.026653\n",
      "\n",
      "LUBZ\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.030519\n",
      "Test avg mae: 0.160000\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.032446\n",
      "Test avg mae: 0.180000\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.031204\n",
      "Test avg mae: 0.160000\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.031441\n",
      "Test avg mae: 0.170000\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.032243\n",
      "Test avg mae: 0.180000\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.033196\n",
      "Test avg mae: 0.175000\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.031445\n",
      "Test avg mae: 0.170000\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.031896\n",
      "Test avg mae: 0.150000\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.031475\n",
      "Test avg mae: 0.160000\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.031954\n",
      "Test avg mae: 0.180000\n",
      "\n",
      "ZUKW2\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024746\n",
      "Test avg mae: 0.035442\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024587\n",
      "Test avg mae: 0.033472\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024649\n",
      "Test avg mae: 0.032509\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024574\n",
      "Test avg mae: 0.033508\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024625\n",
      "Test avg mae: 0.033792\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024838\n",
      "Test avg mae: 0.034519\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024788\n",
      "Test avg mae: 0.036248\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024626\n",
      "Test avg mae: 0.033304\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024571\n",
      "Test avg mae: 0.032324\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024572\n",
      "Test avg mae: 0.031848\n",
      "\n",
      "DABR\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024302\n",
      "Test avg mae: 0.031127\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024232\n",
      "Test avg mae: 0.061970\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024185\n",
      "Test avg mae: 0.032058\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024129\n",
      "Test avg mae: 0.031252\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024226\n",
      "Test avg mae: 0.031566\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024113\n",
      "Test avg mae: 0.030397\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024186\n",
      "Test avg mae: 0.030980\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024138\n",
      "Test avg mae: 0.031460\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024194\n",
      "Test avg mae: 0.031061\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024163\n",
      "Test avg mae: 0.031359\n",
      "\n",
      "PEKW2\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024472\n",
      "Test avg mae: 0.043610\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024489\n",
      "Test avg mae: 0.044439\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024524\n",
      "Test avg mae: 0.043707\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024507\n",
      "Test avg mae: 0.042732\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024472\n",
      "Test avg mae: 0.042488\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024517\n",
      "Test avg mae: 0.042537\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024527\n",
      "Test avg mae: 0.043707\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024483\n",
      "Test avg mae: 0.042585\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024475\n",
      "Test avg mae: 0.044293\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024497\n",
      "Test avg mae: 0.041854\n",
      "\n",
      "KRZY\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025522\n",
      "Test avg mae: 0.054286\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025226\n",
      "Test avg mae: 0.057143\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025248\n",
      "Test avg mae: 0.070000\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025417\n",
      "Test avg mae: 0.047143\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025459\n",
      "Test avg mae: 0.051429\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025202\n",
      "Test avg mae: 0.058571\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025341\n",
      "Test avg mae: 0.060000\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025273\n",
      "Test avg mae: 0.054286\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025228\n",
      "Test avg mae: 0.058571\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025349\n",
      "Test avg mae: 0.048571\n",
      "\n",
      "OBIS\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023926\n",
      "Test avg mae: 0.025517\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023971\n",
      "Test avg mae: 0.026552\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023946\n",
      "Test avg mae: 0.027793\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023967\n",
      "Test avg mae: 0.024828\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023990\n",
      "Test avg mae: 0.027586\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023950\n",
      "Test avg mae: 0.026276\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023965\n",
      "Test avg mae: 0.026345\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023990\n",
      "Test avg mae: 0.025724\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023987\n",
      "Test avg mae: 0.027034\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023955\n",
      "Test avg mae: 0.027241\n",
      "\n",
      "KAZI\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023951\n",
      "Test avg mae: 0.027810\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023934\n",
      "Test avg mae: 0.026762\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023937\n",
      "Test avg mae: 0.027619\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024004\n",
      "Test avg mae: 0.027429\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023971\n",
      "Test avg mae: 0.025429\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023979\n",
      "Test avg mae: 0.027810\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023944\n",
      "Test avg mae: 0.026286\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023980\n",
      "Test avg mae: 0.026762\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023900\n",
      "Test avg mae: 0.026095\n",
      "\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023933\n",
      "Test avg mae: 0.027048\n",
      "\n",
      "KWLC\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "train (12444, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "dev\n",
      "BRDW\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028971\n",
      "Test avg mae: 0.086500\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.029175\n",
      "Test avg mae: 0.086000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027710\n",
      "Test avg mae: 0.085000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027695\n",
      "Test avg mae: 0.084000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028312\n",
      "Test avg mae: 0.087000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027334\n",
      "Test avg mae: 0.083000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027557\n",
      "Test avg mae: 0.088000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028396\n",
      "Test avg mae: 0.093500\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027768\n",
      "Test avg mae: 0.085500\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026457\n",
      "Test avg mae: 0.089000\n",
      "\n",
      "GROD\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025171\n",
      "Test avg mae: 0.048325\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025280\n",
      "Test avg mae: 0.048629\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025265\n",
      "Test avg mae: 0.050152\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025124\n",
      "Test avg mae: 0.044518\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025599\n",
      "Test avg mae: 0.056294\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025292\n",
      "Test avg mae: 0.046548\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025199\n",
      "Test avg mae: 0.048629\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025107\n",
      "Test avg mae: 0.048477\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025218\n",
      "Test avg mae: 0.048782\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025200\n",
      "Test avg mae: 0.050964\n",
      "\n",
      "GUZI\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024414\n",
      "Test avg mae: 0.038017\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024456\n",
      "Test avg mae: 0.037025\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024583\n",
      "Test avg mae: 0.039917\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024552\n",
      "Test avg mae: 0.040496\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024419\n",
      "Test avg mae: 0.037190\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024548\n",
      "Test avg mae: 0.040992\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024619\n",
      "Test avg mae: 0.040413\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024461\n",
      "Test avg mae: 0.038843\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024484\n",
      "Test avg mae: 0.040248\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024423\n",
      "Test avg mae: 0.034628\n",
      "\n",
      "JEDR\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025750\n",
      "Test avg mae: 0.008889\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025724\n",
      "Test avg mae: 0.012222\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024942\n",
      "Test avg mae: 0.015556\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024687\n",
      "Test avg mae: 0.013333\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025583\n",
      "Test avg mae: 0.016667\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026014\n",
      "Test avg mae: 0.018889\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024612\n",
      "Test avg mae: 0.015556\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024265\n",
      "Test avg mae: 0.013333\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024791\n",
      "Test avg mae: 0.008889\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024083\n",
      "Test avg mae: 0.006667\n",
      "\n",
      "MOSK2\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024982\n",
      "Test avg mae: 0.043147\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025065\n",
      "Test avg mae: 0.039442\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025059\n",
      "Test avg mae: 0.045685\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024984\n",
      "Test avg mae: 0.046650\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025198\n",
      "Test avg mae: 0.053655\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024918\n",
      "Test avg mae: 0.049695\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024984\n",
      "Test avg mae: 0.044924\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025159\n",
      "Test avg mae: 0.039645\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025128\n",
      "Test avg mae: 0.045076\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024987\n",
      "Test avg mae: 0.045685\n",
      "\n",
      "NWLU\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025525\n",
      "Test avg mae: 0.036667\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024986\n",
      "Test avg mae: 0.033913\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024918\n",
      "Test avg mae: 0.037609\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025009\n",
      "Test avg mae: 0.035507\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024886\n",
      "Test avg mae: 0.036304\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025350\n",
      "Test avg mae: 0.036159\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025268\n",
      "Test avg mae: 0.037174\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025467\n",
      "Test avg mae: 0.044203\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025363\n",
      "Test avg mae: 0.037101\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025164\n",
      "Test avg mae: 0.035072\n",
      "\n",
      "PCHB\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024568\n",
      "Test avg mae: 0.041707\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024671\n",
      "Test avg mae: 0.043171\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024638\n",
      "Test avg mae: 0.040488\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024669\n",
      "Test avg mae: 0.043537\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024710\n",
      "Test avg mae: 0.042317\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024639\n",
      "Test avg mae: 0.039024\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024660\n",
      "Test avg mae: 0.042439\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024627\n",
      "Test avg mae: 0.041951\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024666\n",
      "Test avg mae: 0.040732\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024709\n",
      "Test avg mae: 0.044878\n",
      "\n",
      "PPOL\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025905\n",
      "Test avg mae: 0.055522\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025925\n",
      "Test avg mae: 0.074627\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026258\n",
      "Test avg mae: 0.071493\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025296\n",
      "Test avg mae: 0.052836\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025339\n",
      "Test avg mae: 0.050000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026370\n",
      "Test avg mae: 0.091791\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025980\n",
      "Test avg mae: 0.073881\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026007\n",
      "Test avg mae: 0.074328\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025985\n",
      "Test avg mae: 0.087910\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025757\n",
      "Test avg mae: 0.058507\n",
      "\n",
      "RUDN\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025465\n",
      "Test avg mae: 0.047500\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025528\n",
      "Test avg mae: 0.049360\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025330\n",
      "Test avg mae: 0.048256\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025654\n",
      "Test avg mae: 0.050872\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025423\n",
      "Test avg mae: 0.049360\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025523\n",
      "Test avg mae: 0.051860\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025558\n",
      "Test avg mae: 0.047791\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025529\n",
      "Test avg mae: 0.048721\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025503\n",
      "Test avg mae: 0.050930\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025489\n",
      "Test avg mae: 0.047965\n",
      "\n",
      "RYNR\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026632\n",
      "Test avg mae: 0.070596\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026046\n",
      "Test avg mae: 0.059868\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026075\n",
      "Test avg mae: 0.061192\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026108\n",
      "Test avg mae: 0.060728\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026853\n",
      "Test avg mae: 0.082980\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026386\n",
      "Test avg mae: 0.061523\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027056\n",
      "Test avg mae: 0.089735\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026100\n",
      "Test avg mae: 0.068874\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026171\n",
      "Test avg mae: 0.069868\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026221\n",
      "Test avg mae: 0.063046\n",
      "\n",
      "RZEC\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "SGOR\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024412\n",
      "Test avg mae: 0.029419\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025008\n",
      "Test avg mae: 0.084194\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025264\n",
      "Test avg mae: 0.085161\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024350\n",
      "Test avg mae: 0.031032\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024544\n",
      "Test avg mae: 0.083613\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024492\n",
      "Test avg mae: 0.083226\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024589\n",
      "Test avg mae: 0.084387\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025363\n",
      "Test avg mae: 0.084452\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024272\n",
      "Test avg mae: 0.031806\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025422\n",
      "Test avg mae: 0.083548\n",
      "\n",
      "TRBC2\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024320\n",
      "Test avg mae: 0.031864\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024446\n",
      "Test avg mae: 0.031864\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024385\n",
      "Test avg mae: 0.031864\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024495\n",
      "Test avg mae: 0.034068\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024545\n",
      "Test avg mae: 0.032712\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024661\n",
      "Test avg mae: 0.034746\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024446\n",
      "Test avg mae: 0.030339\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024298\n",
      "Test avg mae: 0.031356\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024805\n",
      "Test avg mae: 0.033390\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024428\n",
      "Test avg mae: 0.033390\n",
      "\n",
      "TRN2\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024707\n",
      "Test avg mae: 0.039503\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024751\n",
      "Test avg mae: 0.040442\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024843\n",
      "Test avg mae: 0.039613\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024716\n",
      "Test avg mae: 0.041713\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024761\n",
      "Test avg mae: 0.039171\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024794\n",
      "Test avg mae: 0.041160\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024743\n",
      "Test avg mae: 0.040497\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024749\n",
      "Test avg mae: 0.040994\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024759\n",
      "Test avg mae: 0.040166\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024745\n",
      "Test avg mae: 0.040331\n",
      "\n",
      "TRZS\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026324\n",
      "Test avg mae: 0.089741\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025970\n",
      "Test avg mae: 0.041897\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026548\n",
      "Test avg mae: 0.179310\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026197\n",
      "Test avg mae: 0.038534\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026940\n",
      "Test avg mae: 0.093362\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026428\n",
      "Test avg mae: 0.177241\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026037\n",
      "Test avg mae: 0.177414\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026034\n",
      "Test avg mae: 0.090948\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026646\n",
      "Test avg mae: 0.176897\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026593\n",
      "Test avg mae: 0.090776\n",
      "\n",
      "ZMST\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024322\n",
      "Test avg mae: 0.029791\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024131\n",
      "Test avg mae: 0.028063\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024258\n",
      "Test avg mae: 0.029215\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024214\n",
      "Test avg mae: 0.030157\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024274\n",
      "Test avg mae: 0.031518\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024378\n",
      "Test avg mae: 0.034660\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024129\n",
      "Test avg mae: 0.027958\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024210\n",
      "Test avg mae: 0.029581\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024291\n",
      "Test avg mae: 0.034136\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024350\n",
      "Test avg mae: 0.030052\n",
      "\n",
      "LUBW\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.029929\n",
      "Test avg mae: 0.117917\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.029545\n",
      "Test avg mae: 0.113333\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.030242\n",
      "Test avg mae: 0.134583\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028370\n",
      "Test avg mae: 0.089167\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028703\n",
      "Test avg mae: 0.119583\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.030046\n",
      "Test avg mae: 0.122083\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.030243\n",
      "Test avg mae: 0.130833\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.030052\n",
      "Test avg mae: 0.123750\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027626\n",
      "Test avg mae: 0.077917\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028850\n",
      "Test avg mae: 0.120833\n",
      "\n",
      "DWOL\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024480\n",
      "Test avg mae: 0.048835\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024443\n",
      "Test avg mae: 0.031650\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024452\n",
      "Test avg mae: 0.033495\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024461\n",
      "Test avg mae: 0.032136\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024591\n",
      "Test avg mae: 0.048641\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024491\n",
      "Test avg mae: 0.032282\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024380\n",
      "Test avg mae: 0.032476\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024482\n",
      "Test avg mae: 0.049563\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024437\n",
      "Test avg mae: 0.032913\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024607\n",
      "Test avg mae: 0.049029\n",
      "\n",
      "LUBZ\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023679\n",
      "Test avg mae: 0.020000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023553\n",
      "Test avg mae: 0.006667\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023672\n",
      "Test avg mae: 0.026667\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023809\n",
      "Test avg mae: 0.013333\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023596\n",
      "Test avg mae: 0.006667\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023617\n",
      "Test avg mae: 0.010000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023756\n",
      "Test avg mae: 0.020000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023778\n",
      "Test avg mae: 0.016667\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023575\n",
      "Test avg mae: 0.010000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023789\n",
      "Test avg mae: 0.020000\n",
      "\n",
      "ZUKW2\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024717\n",
      "Test avg mae: 0.036456\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024627\n",
      "Test avg mae: 0.035534\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024642\n",
      "Test avg mae: 0.035194\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024675\n",
      "Test avg mae: 0.038835\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024625\n",
      "Test avg mae: 0.036359\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024703\n",
      "Test avg mae: 0.037282\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024708\n",
      "Test avg mae: 0.037961\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024797\n",
      "Test avg mae: 0.036650\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024798\n",
      "Test avg mae: 0.053786\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024559\n",
      "Test avg mae: 0.036650\n",
      "\n",
      "DABR\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024410\n",
      "Test avg mae: 0.036832\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024548\n",
      "Test avg mae: 0.036460\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024549\n",
      "Test avg mae: 0.034596\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024566\n",
      "Test avg mae: 0.036708\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024553\n",
      "Test avg mae: 0.036273\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024526\n",
      "Test avg mae: 0.037143\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024570\n",
      "Test avg mae: 0.037143\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024582\n",
      "Test avg mae: 0.036708\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024457\n",
      "Test avg mae: 0.036708\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024530\n",
      "Test avg mae: 0.036584\n",
      "\n",
      "PEKW2\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026632\n",
      "Test avg mae: 0.045361\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026724\n",
      "Test avg mae: 0.047938\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026484\n",
      "Test avg mae: 0.046392\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026670\n",
      "Test avg mae: 0.048247\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026608\n",
      "Test avg mae: 0.045464\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026605\n",
      "Test avg mae: 0.046804\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026637\n",
      "Test avg mae: 0.047629\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026723\n",
      "Test avg mae: 0.046804\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026733\n",
      "Test avg mae: 0.047010\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026889\n",
      "Test avg mae: 0.058454\n",
      "\n",
      "KRZY\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.038278\n",
      "Test avg mae: 0.110000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.038771\n",
      "Test avg mae: 0.097273\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.038392\n",
      "Test avg mae: 0.099091\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.038522\n",
      "Test avg mae: 0.104545\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.038693\n",
      "Test avg mae: 0.104545\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.038756\n",
      "Test avg mae: 0.096364\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.038586\n",
      "Test avg mae: 0.099091\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.038925\n",
      "Test avg mae: 0.103636\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.038687\n",
      "Test avg mae: 0.098182\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.038287\n",
      "Test avg mae: 0.106364\n",
      "\n",
      "OBIS\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024280\n",
      "Test avg mae: 0.027320\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024778\n",
      "Test avg mae: 0.115155\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024274\n",
      "Test avg mae: 0.027938\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024968\n",
      "Test avg mae: 0.113505\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024193\n",
      "Test avg mae: 0.027938\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024322\n",
      "Test avg mae: 0.027526\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024552\n",
      "Test avg mae: 0.113608\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024237\n",
      "Test avg mae: 0.028041\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024345\n",
      "Test avg mae: 0.027732\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024254\n",
      "Test avg mae: 0.028763\n",
      "\n",
      "KAZI\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024294\n",
      "Test avg mae: 0.029000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024415\n",
      "Test avg mae: 0.029909\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024394\n",
      "Test avg mae: 0.030636\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024499\n",
      "Test avg mae: 0.028727\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024466\n",
      "Test avg mae: 0.030636\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024289\n",
      "Test avg mae: 0.029364\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024452\n",
      "Test avg mae: 0.028545\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024398\n",
      "Test avg mae: 0.030091\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024537\n",
      "Test avg mae: 0.029364\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024418\n",
      "Test avg mae: 0.031091\n",
      "\n",
      "KWLC\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023724\n",
      "Test avg mae: 0.030000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023606\n",
      "Test avg mae: 0.015000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023589\n",
      "Test avg mae: 0.010000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023717\n",
      "Test avg mae: 0.020000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023802\n",
      "Test avg mae: 0.025000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023751\n",
      "Test avg mae: 0.025000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023677\n",
      "Test avg mae: 0.010000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024050\n",
      "Test avg mae: 0.040000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023581\n",
      "Test avg mae: 0.020000\n",
      "\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023595\n",
      "Test avg mae: 0.015000\n",
      "\n",
      "test\n",
      "BRDW\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025934\n",
      "Test avg mae: 0.060896\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026379\n",
      "Test avg mae: 0.073284\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026053\n",
      "Test avg mae: 0.069552\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026029\n",
      "Test avg mae: 0.060448\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026420\n",
      "Test avg mae: 0.058358\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026025\n",
      "Test avg mae: 0.064627\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025910\n",
      "Test avg mae: 0.060448\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026421\n",
      "Test avg mae: 0.086119\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026229\n",
      "Test avg mae: 0.059552\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026069\n",
      "Test avg mae: 0.061045\n",
      "\n",
      "GROD\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025325\n",
      "Test avg mae: 0.044286\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025248\n",
      "Test avg mae: 0.044286\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025428\n",
      "Test avg mae: 0.047029\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025281\n",
      "Test avg mae: 0.042686\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025347\n",
      "Test avg mae: 0.047200\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025475\n",
      "Test avg mae: 0.048343\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025273\n",
      "Test avg mae: 0.060286\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025299\n",
      "Test avg mae: 0.046743\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025243\n",
      "Test avg mae: 0.045829\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025070\n",
      "Test avg mae: 0.043600\n",
      "\n",
      "GUZI\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025129\n",
      "Test avg mae: 0.040556\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025515\n",
      "Test avg mae: 0.050556\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025496\n",
      "Test avg mae: 0.049841\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025842\n",
      "Test avg mae: 0.050238\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025786\n",
      "Test avg mae: 0.058254\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025247\n",
      "Test avg mae: 0.042222\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025569\n",
      "Test avg mae: 0.049841\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025710\n",
      "Test avg mae: 0.049048\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025632\n",
      "Test avg mae: 0.050476\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025607\n",
      "Test avg mae: 0.059206\n",
      "\n",
      "JEDR\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027847\n",
      "Test avg mae: 0.201290\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026951\n",
      "Test avg mae: 0.043387\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027874\n",
      "Test avg mae: 0.197258\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027162\n",
      "Test avg mae: 0.042581\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026982\n",
      "Test avg mae: 0.040161\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027034\n",
      "Test avg mae: 0.042097\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027053\n",
      "Test avg mae: 0.043710\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027056\n",
      "Test avg mae: 0.041774\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027195\n",
      "Test avg mae: 0.042903\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026985\n",
      "Test avg mae: 0.043548\n",
      "\n",
      "MOSK2\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024563\n",
      "Test avg mae: 0.036432\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024515\n",
      "Test avg mae: 0.035276\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024500\n",
      "Test avg mae: 0.035276\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024548\n",
      "Test avg mae: 0.034724\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024531\n",
      "Test avg mae: 0.035678\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024597\n",
      "Test avg mae: 0.035779\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024430\n",
      "Test avg mae: 0.035879\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024420\n",
      "Test avg mae: 0.033618\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024539\n",
      "Test avg mae: 0.035377\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024518\n",
      "Test avg mae: 0.035327\n",
      "\n",
      "NWLU\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024630\n",
      "Test avg mae: 0.031074\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024590\n",
      "Test avg mae: 0.030872\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024608\n",
      "Test avg mae: 0.030268\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024637\n",
      "Test avg mae: 0.028591\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024670\n",
      "Test avg mae: 0.029866\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024449\n",
      "Test avg mae: 0.029597\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024694\n",
      "Test avg mae: 0.030201\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024591\n",
      "Test avg mae: 0.031544\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024664\n",
      "Test avg mae: 0.029396\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024568\n",
      "Test avg mae: 0.030470\n",
      "\n",
      "PCHB\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024545\n",
      "Test avg mae: 0.037215\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024679\n",
      "Test avg mae: 0.035443\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024926\n",
      "Test avg mae: 0.045190\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024639\n",
      "Test avg mae: 0.035696\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024640\n",
      "Test avg mae: 0.036582\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024553\n",
      "Test avg mae: 0.033797\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024715\n",
      "Test avg mae: 0.044557\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024856\n",
      "Test avg mae: 0.045949\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024775\n",
      "Test avg mae: 0.046076\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024738\n",
      "Test avg mae: 0.035570\n",
      "\n",
      "PPOL\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025437\n",
      "Test avg mae: 0.048475\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025318\n",
      "Test avg mae: 0.050000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025423\n",
      "Test avg mae: 0.049661\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025431\n",
      "Test avg mae: 0.049153\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025138\n",
      "Test avg mae: 0.048305\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025164\n",
      "Test avg mae: 0.048475\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025158\n",
      "Test avg mae: 0.041695\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025334\n",
      "Test avg mae: 0.050847\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025233\n",
      "Test avg mae: 0.048983\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025176\n",
      "Test avg mae: 0.048644\n",
      "\n",
      "RUDN\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025884\n",
      "Test avg mae: 0.054605\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025868\n",
      "Test avg mae: 0.059211\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026080\n",
      "Test avg mae: 0.057566\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025783\n",
      "Test avg mae: 0.058158\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025573\n",
      "Test avg mae: 0.044605\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026530\n",
      "Test avg mae: 0.055329\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026172\n",
      "Test avg mae: 0.057434\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026277\n",
      "Test avg mae: 0.057303\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026563\n",
      "Test avg mae: 0.058487\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026182\n",
      "Test avg mae: 0.055592\n",
      "\n",
      "RYNR\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027648\n",
      "Test avg mae: 0.086963\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027588\n",
      "Test avg mae: 0.082815\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027658\n",
      "Test avg mae: 0.089778\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027625\n",
      "Test avg mae: 0.108296\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026857\n",
      "Test avg mae: 0.079926\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027323\n",
      "Test avg mae: 0.098370\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026899\n",
      "Test avg mae: 0.076296\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027445\n",
      "Test avg mae: 0.077481\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027287\n",
      "Test avg mae: 0.086963\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026892\n",
      "Test avg mae: 0.084148\n",
      "\n",
      "RZEC\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "division by zero\n",
      "SGOR\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025765\n",
      "Test avg mae: 0.042595\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025846\n",
      "Test avg mae: 0.042848\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025771\n",
      "Test avg mae: 0.043861\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026033\n",
      "Test avg mae: 0.042532\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025628\n",
      "Test avg mae: 0.043734\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025968\n",
      "Test avg mae: 0.043987\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025798\n",
      "Test avg mae: 0.042848\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026101\n",
      "Test avg mae: 0.045063\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025852\n",
      "Test avg mae: 0.042468\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025887\n",
      "Test avg mae: 0.044241\n",
      "\n",
      "TRBC2\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024363\n",
      "Test avg mae: 0.033091\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024329\n",
      "Test avg mae: 0.034000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024381\n",
      "Test avg mae: 0.032000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024374\n",
      "Test avg mae: 0.032545\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024416\n",
      "Test avg mae: 0.036182\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024293\n",
      "Test avg mae: 0.031273\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024474\n",
      "Test avg mae: 0.036727\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024390\n",
      "Test avg mae: 0.036182\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024293\n",
      "Test avg mae: 0.031818\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024265\n",
      "Test avg mae: 0.033273\n",
      "\n",
      "TRN2\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027654\n",
      "Test avg mae: 0.063333\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027777\n",
      "Test avg mae: 0.060864\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027985\n",
      "Test avg mae: 0.072531\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.028154\n",
      "Test avg mae: 0.072840\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027799\n",
      "Test avg mae: 0.070802\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027817\n",
      "Test avg mae: 0.064198\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027730\n",
      "Test avg mae: 0.064506\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027682\n",
      "Test avg mae: 0.068889\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027775\n",
      "Test avg mae: 0.064444\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.027888\n",
      "Test avg mae: 0.071111\n",
      "\n",
      "TRZS\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024738\n",
      "Test avg mae: 0.037113\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024839\n",
      "Test avg mae: 0.036701\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024709\n",
      "Test avg mae: 0.036082\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024755\n",
      "Test avg mae: 0.035773\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024726\n",
      "Test avg mae: 0.035567\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024823\n",
      "Test avg mae: 0.036701\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024761\n",
      "Test avg mae: 0.035052\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024710\n",
      "Test avg mae: 0.033918\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024731\n",
      "Test avg mae: 0.038454\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024848\n",
      "Test avg mae: 0.036186\n",
      "\n",
      "ZMST\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025525\n",
      "Test avg mae: 0.040159\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025413\n",
      "Test avg mae: 0.036667\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025533\n",
      "Test avg mae: 0.038624\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025489\n",
      "Test avg mae: 0.038254\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025375\n",
      "Test avg mae: 0.037249\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025236\n",
      "Test avg mae: 0.037513\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025373\n",
      "Test avg mae: 0.037354\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025520\n",
      "Test avg mae: 0.051640\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025388\n",
      "Test avg mae: 0.037566\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025274\n",
      "Test avg mae: 0.038201\n",
      "\n",
      "LUBW\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025608\n",
      "Test avg mae: 0.063000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025088\n",
      "Test avg mae: 0.042000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025370\n",
      "Test avg mae: 0.044000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025287\n",
      "Test avg mae: 0.055000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024473\n",
      "Test avg mae: 0.033000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024490\n",
      "Test avg mae: 0.039000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026141\n",
      "Test avg mae: 0.061000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025247\n",
      "Test avg mae: 0.048000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025590\n",
      "Test avg mae: 0.050000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025214\n",
      "Test avg mae: 0.046000\n",
      "\n",
      "DWOL\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024828\n",
      "Test avg mae: 0.036150\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024786\n",
      "Test avg mae: 0.037000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024687\n",
      "Test avg mae: 0.035150\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024730\n",
      "Test avg mae: 0.036050\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024863\n",
      "Test avg mae: 0.036000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024617\n",
      "Test avg mae: 0.037150\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024813\n",
      "Test avg mae: 0.038150\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024747\n",
      "Test avg mae: 0.035500\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024657\n",
      "Test avg mae: 0.036600\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024653\n",
      "Test avg mae: 0.036100\n",
      "\n",
      "LUBZ\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023514\n",
      "Test avg mae: 0.020000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023465\n",
      "Test avg mae: 0.010000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023750\n",
      "Test avg mae: 0.030000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023617\n",
      "Test avg mae: 0.000000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023518\n",
      "Test avg mae: 0.000000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023535\n",
      "Test avg mae: 0.010000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023519\n",
      "Test avg mae: 0.000000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023594\n",
      "Test avg mae: 0.010000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023488\n",
      "Test avg mae: 0.010000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.023619\n",
      "Test avg mae: 0.010000\n",
      "\n",
      "ZUKW2\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024833\n",
      "Test avg mae: 0.034316\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024815\n",
      "Test avg mae: 0.034789\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024881\n",
      "Test avg mae: 0.034526\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024811\n",
      "Test avg mae: 0.035632\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024757\n",
      "Test avg mae: 0.032316\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024949\n",
      "Test avg mae: 0.034000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024804\n",
      "Test avg mae: 0.034474\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024805\n",
      "Test avg mae: 0.034789\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024807\n",
      "Test avg mae: 0.036105\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024812\n",
      "Test avg mae: 0.035316\n",
      "\n",
      "DABR\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024772\n",
      "Test avg mae: 0.039000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025039\n",
      "Test avg mae: 0.040500\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025131\n",
      "Test avg mae: 0.037250\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025016\n",
      "Test avg mae: 0.038500\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025114\n",
      "Test avg mae: 0.039125\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024994\n",
      "Test avg mae: 0.038688\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025277\n",
      "Test avg mae: 0.038375\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024936\n",
      "Test avg mae: 0.038125\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024811\n",
      "Test avg mae: 0.038500\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025205\n",
      "Test avg mae: 0.039000\n",
      "\n",
      "PEKW2\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024234\n",
      "Test avg mae: 0.036897\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024220\n",
      "Test avg mae: 0.037816\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024231\n",
      "Test avg mae: 0.037931\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024209\n",
      "Test avg mae: 0.036207\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024214\n",
      "Test avg mae: 0.037241\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024192\n",
      "Test avg mae: 0.038966\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024206\n",
      "Test avg mae: 0.036207\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024220\n",
      "Test avg mae: 0.034828\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024178\n",
      "Test avg mae: 0.037126\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024201\n",
      "Test avg mae: 0.040460\n",
      "\n",
      "KRZY\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025639\n",
      "Test avg mae: 0.070000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024752\n",
      "Test avg mae: 0.050000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024743\n",
      "Test avg mae: 0.060000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025550\n",
      "Test avg mae: 0.070000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025862\n",
      "Test avg mae: 0.060000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024733\n",
      "Test avg mae: 0.050000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025121\n",
      "Test avg mae: 0.060000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026550\n",
      "Test avg mae: 0.080000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.024653\n",
      "Test avg mae: 0.050000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025185\n",
      "Test avg mae: 0.070000\n",
      "\n",
      "OBIS\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025883\n",
      "Test avg mae: 0.054245\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026269\n",
      "Test avg mae: 0.054717\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025488\n",
      "Test avg mae: 0.054245\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025862\n",
      "Test avg mae: 0.042830\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025587\n",
      "Test avg mae: 0.056509\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.026170\n",
      "Test avg mae: 0.055566\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025503\n",
      "Test avg mae: 0.057925\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025792\n",
      "Test avg mae: 0.055755\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025840\n",
      "Test avg mae: 0.052736\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025825\n",
      "Test avg mae: 0.056415\n",
      "\n",
      "KAZI\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025883\n",
      "Test avg mae: 0.044649\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025872\n",
      "Test avg mae: 0.042982\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025842\n",
      "Test avg mae: 0.042281\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025850\n",
      "Test avg mae: 0.040789\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025822\n",
      "Test avg mae: 0.041930\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025798\n",
      "Test avg mae: 0.041491\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025872\n",
      "Test avg mae: 0.041404\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025900\n",
      "Test avg mae: 0.041316\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025843\n",
      "Test avg mae: 0.042632\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025867\n",
      "Test avg mae: 0.042368\n",
      "\n",
      "KWLC\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025280\n",
      "Test avg mae: 0.051923\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025302\n",
      "Test avg mae: 0.050000\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025410\n",
      "Test avg mae: 0.053462\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025195\n",
      "Test avg mae: 0.048269\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025163\n",
      "Test avg mae: 0.049423\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025168\n",
      "Test avg mae: 0.048269\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025206\n",
      "Test avg mae: 0.050385\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025152\n",
      "Test avg mae: 0.048846\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025373\n",
      "Test avg mae: 0.054615\n",
      "\n",
      "test (2785, 17) 100\n",
      "using random window\n",
      "Test avg loss: 0.025162\n",
      "Test avg mae: 0.050192\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "stations = data.metadata.station_code.unique()\n",
    "\n",
    "results = []\n",
    "\n",
    "for split in splits: \n",
    "    split_results = {}\n",
    "    print(split)\n",
    "    for station in stations: \n",
    "        print(station)\n",
    "        split_results[station] = {'mae':[], 'loss':[]}\n",
    "        for i in range(10):\n",
    "            gen = train.get_data_generator(split=split, station=station, sampling_rate=sampling_rate, path=data_path)\n",
    "            data_loader = DataLoader(gen, batch_size=256, shuffle=False, num_workers=0,\n",
    "                                    worker_init_fn=worker_seeding)\n",
    "        \n",
    "            test_loss, test_mae = None, None\n",
    "            try: \n",
    "                test_loss, test_mae = train.test_one_epoch(model, data_loader, pick_mae, wandb_log=False)\n",
    "                test_mae = float(test_mae)\n",
    "        \n",
    "            except Exception as e: \n",
    "                print(e)\n",
    "        \n",
    "            split_results[station]['mae'].append(test_mae)\n",
    "            split_results[station]['loss'].append(test_loss)\n",
    "    results.append(split_results)\n",
    "        \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "206e6973-61bb-4f84-95a5-f910c7c7dc2b",
   "metadata": {},
   "source": [
    "#### Plot results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f0366dc6-d1e1-44cb-b748-d0af44a77219",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3464: RuntimeWarning: Mean of empty slice.\n",
      "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
      "/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3464: RuntimeWarning: Mean of empty slice.\n",
      "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
      "/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3464: RuntimeWarning: Mean of empty slice.\n",
      "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
      "/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3464: RuntimeWarning: Mean of empty slice.\n",
      "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
      "/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3464: RuntimeWarning: Mean of empty slice.\n",
      "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
      "/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3464: RuntimeWarning: Mean of empty slice.\n",
      "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
      "/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 48804 characters]",
      "text/plain": [
       "<Figure size 1500x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 47640 characters]",
      "text/plain": [
       "<Figure size 1500x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 45972 characters]",
      "text/plain": [
       "<Figure size 1500x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results_df = []\n",
    "for i, split in enumerate(splits): \n",
    "    df = pd.DataFrame(results[i]).transpose()\n",
    "\n",
    "    for station, values in df[['mae']].itertuples():\n",
    "        df.loc[station, 'mae'] =  np.mean([v for v in values if v is not None]) \n",
    "    for station, values in df[['loss']].itertuples():\n",
    "        df.loc[station, 'loss'] =  np.mean([v for v in values if v is not None]) \n",
    "        \n",
    "    df.plot(kind='bar', figsize=(15,4), title=f\"Mean results per station in {split} set\")\n",
    "\n",
    "    results_df.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "14171ea5-5438-4e57-8e1d-ed30df68340b",
   "metadata": {},
   "outputs": [],
   "source": [
    "### Check correlation between trainin data size and obtained results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d2bec214-3058-4337-848b-035dc2395c76",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = frames_per_station.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c8718b24-7257-4bc5-b6a4-da68012dafac",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats['train_mae'] = results_df[0]['mae']\n",
    "stats['dev_mae'] = results_df[1]['mae']\n",
    "stats['test_mae'] = results_df[2]['mae']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "30cc267a-8b2c-4b72-9e55-19e87874ebcb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='train', ylabel='train_mae'>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 19280 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "stats.plot(kind='scatter', x ='train', y='train_mae')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ebac708b-c3c2-4f08-9574-4c6daffa96bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train</th>\n",
       "      <th>dev</th>\n",
       "      <th>test</th>\n",
       "      <th>train_mae</th>\n",
       "      <th>dev_mae</th>\n",
       "      <th>test_mae</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>station_code</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BRDW</th>\n",
       "      <td>160.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>0.092019</td>\n",
       "      <td>0.08675</td>\n",
       "      <td>0.065433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DABR</th>\n",
       "      <td>359.0</td>\n",
       "      <td>161.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>0.034323</td>\n",
       "      <td>0.036516</td>\n",
       "      <td>0.038706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DWOL</th>\n",
       "      <td>479.0</td>\n",
       "      <td>206.0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>0.02986</td>\n",
       "      <td>0.039102</td>\n",
       "      <td>0.036385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GROD</th>\n",
       "      <td>1052.0</td>\n",
       "      <td>197.0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>0.05752</td>\n",
       "      <td>0.049132</td>\n",
       "      <td>0.047029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GUZI</th>\n",
       "      <td>740.0</td>\n",
       "      <td>121.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>0.035936</td>\n",
       "      <td>0.038777</td>\n",
       "      <td>0.050024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JEDR</th>\n",
       "      <td>809.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>0.035391</td>\n",
       "      <td>0.013</td>\n",
       "      <td>0.073871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KAZI</th>\n",
       "      <td>105.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>114.0</td>\n",
       "      <td>0.026905</td>\n",
       "      <td>0.029736</td>\n",
       "      <td>0.042184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KRZY</th>\n",
       "      <td>7.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.056</td>\n",
       "      <td>0.101909</td>\n",
       "      <td>0.062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LUBW</th>\n",
       "      <td>33.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.090697</td>\n",
       "      <td>0.115</td>\n",
       "      <td>0.0481</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LUBZ</th>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.1685</td>\n",
       "      <td>0.015</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MOSK2</th>\n",
       "      <td>958.0</td>\n",
       "      <td>197.0</td>\n",
       "      <td>199.0</td>\n",
       "      <td>0.040543</td>\n",
       "      <td>0.04536</td>\n",
       "      <td>0.035337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NWLU</th>\n",
       "      <td>902.0</td>\n",
       "      <td>138.0</td>\n",
       "      <td>149.0</td>\n",
       "      <td>0.03412</td>\n",
       "      <td>0.036971</td>\n",
       "      <td>0.030188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>OBIS</th>\n",
       "      <td>145.0</td>\n",
       "      <td>97.0</td>\n",
       "      <td>106.0</td>\n",
       "      <td>0.02649</td>\n",
       "      <td>0.053753</td>\n",
       "      <td>0.054094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PCHB</th>\n",
       "      <td>420.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>0.040777</td>\n",
       "      <td>0.042024</td>\n",
       "      <td>0.039608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PEKW2</th>\n",
       "      <td>205.0</td>\n",
       "      <td>97.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.043195</td>\n",
       "      <td>0.04801</td>\n",
       "      <td>0.037368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PPOL</th>\n",
       "      <td>463.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>0.064004</td>\n",
       "      <td>0.06909</td>\n",
       "      <td>0.048424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RUDN</th>\n",
       "      <td>941.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>152.0</td>\n",
       "      <td>0.047671</td>\n",
       "      <td>0.049262</td>\n",
       "      <td>0.055829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RYNR</th>\n",
       "      <td>874.0</td>\n",
       "      <td>151.0</td>\n",
       "      <td>135.0</td>\n",
       "      <td>0.065755</td>\n",
       "      <td>0.068841</td>\n",
       "      <td>0.087104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RZEC</th>\n",
       "      <td>29.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.028448</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SGOR</th>\n",
       "      <td>845.0</td>\n",
       "      <td>155.0</td>\n",
       "      <td>158.0</td>\n",
       "      <td>0.036054</td>\n",
       "      <td>0.068084</td>\n",
       "      <td>0.043418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TRBC2</th>\n",
       "      <td>295.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>0.044679</td>\n",
       "      <td>0.032559</td>\n",
       "      <td>0.033709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TRN2</th>\n",
       "      <td>1020.0</td>\n",
       "      <td>181.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>0.053552</td>\n",
       "      <td>0.040359</td>\n",
       "      <td>0.067352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TRZS</th>\n",
       "      <td>209.0</td>\n",
       "      <td>116.0</td>\n",
       "      <td>97.0</td>\n",
       "      <td>0.044144</td>\n",
       "      <td>0.115612</td>\n",
       "      <td>0.036155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ZMST</th>\n",
       "      <td>1084.0</td>\n",
       "      <td>191.0</td>\n",
       "      <td>189.0</td>\n",
       "      <td>0.049537</td>\n",
       "      <td>0.030513</td>\n",
       "      <td>0.039323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ZUKW2</th>\n",
       "      <td>308.0</td>\n",
       "      <td>206.0</td>\n",
       "      <td>190.0</td>\n",
       "      <td>0.033697</td>\n",
       "      <td>0.038471</td>\n",
       "      <td>0.034626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KWLC</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.021</td>\n",
       "      <td>0.050538</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               train    dev   test train_mae   dev_mae  test_mae\n",
       "station_code                                                    \n",
       "BRDW           160.0   20.0   67.0  0.092019   0.08675  0.065433\n",
       "DABR           359.0  161.0  160.0  0.034323  0.036516  0.038706\n",
       "DWOL           479.0  206.0  200.0   0.02986  0.039102  0.036385\n",
       "GROD          1052.0  197.0  175.0   0.05752  0.049132  0.047029\n",
       "GUZI           740.0  121.0  126.0  0.035936  0.038777  0.050024\n",
       "JEDR           809.0    9.0   62.0  0.035391     0.013  0.073871\n",
       "KAZI           105.0  110.0  114.0  0.026905  0.029736  0.042184\n",
       "KRZY             7.0   11.0    1.0     0.056  0.101909     0.062\n",
       "LUBW            33.0   24.0   10.0  0.090697     0.115    0.0481\n",
       "LUBZ             2.0    3.0    1.0    0.1685     0.015      0.01\n",
       "MOSK2          958.0  197.0  199.0  0.040543   0.04536  0.035337\n",
       "NWLU           902.0  138.0  149.0   0.03412  0.036971  0.030188\n",
       "OBIS           145.0   97.0  106.0   0.02649  0.053753  0.054094\n",
       "PCHB           420.0   82.0   79.0  0.040777  0.042024  0.039608\n",
       "PEKW2          205.0   97.0   87.0  0.043195   0.04801  0.037368\n",
       "PPOL           463.0   67.0   59.0  0.064004   0.06909  0.048424\n",
       "RUDN           941.0  172.0  152.0  0.047671  0.049262  0.055829\n",
       "RYNR           874.0  151.0  135.0  0.065755  0.068841  0.087104\n",
       "RZEC            29.0    NaN    NaN  0.028448       NaN       NaN\n",
       "SGOR           845.0  155.0  158.0  0.036054  0.068084  0.043418\n",
       "TRBC2          295.0   59.0   55.0  0.044679  0.032559  0.033709\n",
       "TRN2          1020.0  181.0  162.0  0.053552  0.040359  0.067352\n",
       "TRZS           209.0  116.0   97.0  0.044144  0.115612  0.036155\n",
       "ZMST          1084.0  191.0  189.0  0.049537  0.030513  0.039323\n",
       "ZUKW2          308.0  206.0  190.0  0.033697  0.038471  0.034626\n",
       "KWLC             NaN    2.0   52.0       NaN     0.021  0.050538"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f02c9173-986c-45d5-973c-a68406ff20df",
   "metadata": {},
   "source": [
    "### Check predictions for stations with highest MAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "66a99163-04cd-4ecb-a3c8-fa8dc112c435",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "highest mean MAE in dev set: 0.12 for station: TRZS\n",
      "highest mean MAE in test set: 0.09 for station: RYNR\n"
     ]
    }
   ],
   "source": [
    "dev_res = results_df[1]\n",
    "station_with_worst_res_dev_set = dev_res[dev_res.mae == dev_res.mae.max()].index[0]\n",
    "highest_dev_mae = dev_res.loc[station_with_worst_res_dev_set, 'mae']\n",
    "\n",
    "test_res = results_df[2]\n",
    "station_with_worst_res_test_set = test_res[test_res.mae == test_res.mae.max()].index[0]\n",
    "highest_test_mae = test_res.loc[station_with_worst_res_test_set, 'mae']\n",
    "print(f\"highest mean MAE in dev set: {highest_dev_mae:.2f} for station: {station_with_worst_res_dev_set}\")\n",
    "print(f\"highest mean MAE in test set: {highest_test_mae:.2f} for station: {station_with_worst_res_test_set}\")\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "73a07433-1663-440c-bce3-d73988e9685b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_sample(sample, model, i, desc=None): \n",
    "    fig = plt.figure(figsize=(15, 10))\n",
    "    \n",
    "    axs = fig.subplots(2, 1, sharex=True, gridspec_kw={\"hspace\": 0, \"height_ratios\": [3,  2]})\n",
    "    axs[0].plot(sample[\"X\"][0].T, label='x')\n",
    "    plt.legend()\n",
    "    axs[1].plot(sample[\"y\"][0].T, label='y')\n",
    "    \n",
    "    model.eval()  # close the model for evaluation\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        pred = model(torch.tensor(sample[\"X\"], device=model.device).unsqueeze(0))  # Add a fake batch dimension\n",
    "        pred = pred[0].cpu().numpy()\n",
    "        \n",
    "        axs[1].plot(pred[0], label='pred', color='orange')\n",
    "        plt.legend()\n",
    "\n",
    "        pred_pick_idx  = np.argmax(pred[0])\n",
    "        true_pick_idx = np.argmax(sample['y'][0])\n",
    "\n",
    "        \n",
    "        \n",
    "        mae_error = np.abs(pred_pick_idx - true_pick_idx) /100 #mae in seconds\n",
    "\n",
    "        fig.suptitle(f\"Predictions for sample: {i} {desc}, mae: {mae_error}s\")\n",
    "        \n",
    "        plt.show()\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d28d456a-4034-4eea-ac28-ed29ddb98d75",
   "metadata": {},
   "source": [
    "### Find random samples that reproduce obtained results \n",
    "\n",
    "Results are not deterministic, because samples generator used during training augments samples by introducing random padding, see https://seisbench.readthedocs.io/en/stable/pages/documentation/generate.html?highlight=generate#seisbench.generate.windows.RandomWindow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "235ae13e-3250-49e6-8bb2-765c037b0870",
   "metadata": {},
   "outputs": [],
   "source": [
    "##### dev set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5e3829f5-360a-431f-9fdd-bdeff6a5a1e4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev (2773, 17) 100\n",
      "using random window\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "dev (2773, 17) 100\n",
      "using random window\n",
      "dev (2773, 17) 100\n",
      "using random window\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.594e+01 4.900e-01 4.500e-01 2.600e-01 2.000e-01 1.300e-01 1.200e-01\n",
      " 1.200e-01 1.000e-01 9.000e-02 9.000e-02 8.000e-02 8.000e-02 6.000e-02\n",
      " 6.000e-02 6.000e-02 6.000e-02 6.000e-02 6.000e-02 6.000e-02 5.000e-02\n",
      " 5.000e-02 5.000e-02 5.000e-02 4.000e-02 4.000e-02 4.000e-02 4.000e-02\n",
      " 4.000e-02 4.000e-02 4.000e-02 4.000e-02 4.000e-02 4.000e-02 3.000e-02\n",
      " 3.000e-02 3.000e-02 3.000e-02 3.000e-02 3.000e-02 3.000e-02 3.000e-02\n",
      " 3.000e-02 3.000e-02 3.000e-02 3.000e-02 3.000e-02 3.000e-02 3.000e-02\n",
      " 3.000e-02 3.000e-02 3.000e-02 3.000e-02 2.000e-02 2.000e-02 2.000e-02\n",
      " 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02\n",
      " 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02\n",
      " 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02 1.000e-02\n",
      " 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02\n",
      " 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02\n",
      " 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02\n",
      " 1.000e-02 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 0.000e+00 0.000e+00]\n",
      "59 15.94\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 151936 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.49\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 131816 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "53 0.45\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 142556 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "110 0.26\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 148020 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64 0.2\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 99036 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "mean_mae = 0\n",
    "samples = []\n",
    "split = 'dev'\n",
    "station = station_with_worst_res_dev_set\n",
    "\n",
    "while mean_mae < highest_dev_mae: \n",
    "\n",
    "    gen = train.get_data_generator(split=split, station=station , sampling_rate=sampling_rate, path=data_path)\n",
    "    station_mae = []\n",
    "    with torch.no_grad():\n",
    "        for i in range(len(gen)): \n",
    "            # idx = np.random.randint(len(gen))\n",
    "            idx = i\n",
    "            sample = gen[idx]\n",
    "            samples.append(sample)\n",
    "            pred = model(torch.tensor(sample[\"X\"], device=model.device).unsqueeze(0))  \n",
    "            pred = pred[0].cpu().numpy()\n",
    "    \n",
    "            pred_pick_idx  = np.argmax(pred[0])\n",
    "            true_pick_idx = np.argmax(sample['y'][0])            \n",
    "                \n",
    "            mae_error = np.abs(pred_pick_idx - true_pick_idx) /100 #mae in seconds\n",
    "            station_mae.append(mae_error)\n",
    "    \n",
    "    sorted = np.argsort(station_mae)[::-1]\n",
    "    mean_mae = np.mean(station_mae)\n",
    "\n",
    "print(np.array(station_mae)[sorted])\n",
    "\n",
    "\n",
    "## plot samples with mae error at leas 0.2s\n",
    "for idx in sorted:\n",
    "    if station_mae[idx] < 0.2: \n",
    "        break\n",
    "    print(idx, station_mae[idx])\n",
    "    plot_sample(samples[idx], model, idx, desc=f\" from station {station} {split} set\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "fc074059-8faf-4d7b-8b2c-49995df6fb37",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test (2785, 17) 100\n",
      "using random window\n",
      "test (2785, 17) 100\n",
      "using random window\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 91  40  63 106  28 119 100  33 129 105  42  54  86  32 130  24  61   3\n",
      "  79  73  67  89 128 124  76  50  46  36  71  31   5  83  98 126  18  21\n",
      "  77   2 134 114 127 109 125  22  30  23  20  81  15  43  44  45 113  49\n",
      "  96  51  74  94 117  52 118  95 122  57 123  14  13  12   9   8   7   6\n",
      " 131 132  56  47  65  93  90  97 110  87  75  85 108 102  80  59  25 104\n",
      "  19  58  17  82  84  11  10   4 120   0  68  55  48  69 112  53  38  39\n",
      "  37  60 116  66 111  92 101  88  62  99  64   1  34 133  16  35  70  41\n",
      "  78 103 115  26  27 121 107  72  29]\n",
      "[3.12 2.56 1.23 0.5  0.37 0.23 0.22 0.14 0.13 0.12 0.11 0.1  0.1  0.09\n",
      " 0.09 0.08 0.08 0.08 0.07 0.07 0.07 0.07 0.07 0.06 0.06 0.06 0.06 0.06\n",
      " 0.05 0.05 0.05 0.05 0.05 0.05 0.04 0.04 0.04 0.04 0.04 0.04 0.04 0.04\n",
      " 0.04 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03\n",
      " 0.03 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02\n",
      " 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02\n",
      " 0.02 0.02 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01\n",
      " 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01\n",
      " 0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.\n",
      " 0.   0.   0.   0.   0.   0.   0.   0.   0.  ]\n",
      "91 3.12\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 119040 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40 2.56\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 130552 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63 1.23\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 129584 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106 0.5\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 141068 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28 0.37\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 147032 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "119 0.23\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 95700 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 0.22\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 121612 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "mean_mae = 0\n",
    "samples = []\n",
    "split = 'test'\n",
    "station = station_with_worst_res_test_set\n",
    "\n",
    "while mean_mae < highest_test_mae: \n",
    "\n",
    "    gen = train.get_data_generator(split=split, station=station , sampling_rate=sampling_rate, path=data_path)\n",
    "    station_mae = []\n",
    "    with torch.no_grad():\n",
    "        for i in range(len(gen)): \n",
    "            # idx = np.random.randint(len(gen))\n",
    "            idx = i\n",
    "            sample = gen[idx]\n",
    "            samples.append(sample)\n",
    "            pred = model(torch.tensor(sample[\"X\"], device=model.device).unsqueeze(0))  \n",
    "            pred = pred[0].cpu().numpy()\n",
    "    \n",
    "            pred_pick_idx  = np.argmax(pred[0])\n",
    "            true_pick_idx = np.argmax(sample['y'][0])            \n",
    "                \n",
    "            mae_error = np.abs(pred_pick_idx - true_pick_idx) /100 #mae in seconds\n",
    "            station_mae.append(mae_error)\n",
    "    \n",
    "    sorted = np.argsort(station_mae)[::-1]\n",
    "    mean_mae = np.mean(station_mae)\n",
    "\n",
    "print(sorted)\n",
    "print(np.array(station_mae)[sorted])\n",
    "\n",
    "\n",
    "## plot samples with mae error at leas 0.2s\n",
    "for idx in sorted:\n",
    "    if station_mae[idx] < 0.2: \n",
    "        break\n",
    "    print(idx, station_mae[idx])\n",
    "    plot_sample(samples[idx], model, idx, desc=f\" from station {station} {split} set\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7e599f32-b888-4235-8d15-f9d2c98a59cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index                                            30312\n",
       "source_origin_time             2021-08-24 03:01:23.500\n",
       "source_latitude_deg                      5714247.36402\n",
       "source_longitude_deg                    5578128.427708\n",
       "source_depth_km                                    0.8\n",
       "source_magnitude                              1.522222\n",
       "split                                             test\n",
       "station_network_code                                PL\n",
       "station_code                                     MOSK2\n",
       "trace_channel                                      EHE\n",
       "trace_sampling_rate_hz                           100.0\n",
       "trace_start_time           2021-08-24T03:01:17.360000Z\n",
       "trace_Pg_arrival_sample                          727.0\n",
       "trace_name                        bucket29$66,:3,:2001\n",
       "trace_Sg_arrival_sample                            NaN\n",
       "trace_chunk                                           \n",
       "trace_component_order                              ZNE\n",
       "Name: 30312, dtype: object"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.test().metadata.iloc[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b3d9f158-5e63-4cc1-8052-844749fe098f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "07561939-653f-441d-96c1-ac8046668838",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test (2785, 17) 100\n",
      "using random window\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 171296 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 133056 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 95656 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 82204 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 136776 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gen = train.get_data_generator(split='test', station=None , sampling_rate=sampling_rate, path=data_path)\n",
    "\n",
    "for i in range(5): \n",
    "    idx = np.random.randint(len(gen))\n",
    "    sample = gen[idx]\n",
    "    \n",
    "    plot_sample(sample, model, idx, desc=\"\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "b9a025cc-15a0-4ae0-a926-15d594a40200",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'X': array([[-2.9614418e-19, -2.9614418e-19, -2.9614418e-19, ...,\n",
       "         -2.9614418e-19, -2.9614418e-19, -2.9614418e-19],\n",
       "        [-6.4017395e-19, -6.4017395e-19, -6.4017395e-19, ...,\n",
       "         -6.4017395e-19, -6.4017395e-19, -6.4017395e-19],\n",
       "        [-1.0255816e-18, -1.0255816e-18, -1.0255816e-18, ...,\n",
       "         -1.0255816e-18, -1.0255816e-18, -1.0255816e-18]], dtype=float32),\n",
       " 'y': array([[2.43596292e-226, 7.12942807e-226, 2.08428048e-225, ...,\n",
       "         0.00000000e+000, 0.00000000e+000, 0.00000000e+000],\n",
       "        [1.00000000e+000, 1.00000000e+000, 1.00000000e+000, ...,\n",
       "         1.00000000e+000, 1.00000000e+000, 1.00000000e+000]])}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "902426eb-6e4d-472f-b38a-bbc94761af83",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
