platform-demo-scripts/notebooks/Check model performance depending on station-random window.ipynb

Check model performance depending on station

Samples generated with random window

In [1]:

import pandas as pd
from obspy.core.event import read_events
import matplotlib.pyplot as plt

import seisbench.models as sbm
import torch
import torch.nn as nn

import seisbench.data as sbd
import seisbench.generate as sbg
import seisbench.models as sbm
from seisbench.util import worker_seeding
import numpy as np
from torch.utils.data import DataLoader
from pathlib import Path
import wandb
import os
import sys

from pathlib import Path
cwd = str(Path.cwd().parent)
sys.path.append(cwd)
from scripts import train

wandb: Currently logged in as: kmilian (epos). Use `wandb login --relogin` to force relogin
wandb: WARNING If you're specifying your api key in code, ensure this code is not shared publicly.
wandb: WARNING Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.
wandb: Appending key for api.wandb.ai to your netrc file: /Users/krystynamilian/.netrc

In [2]:

model = train.load_model()

run = wandb.init()
artifact = run.use_artifact('epos/training_seisbench_models_on_igf_data/model:v113', type='model')
artifact_dir = artifact.download()
fname = artifact_dir + "/" + os.listdir(artifact_dir)[0]

model.load_state_dict(torch.load(fname))
model.eval()

Out [2]:

wandb version 0.15.4 is available! To upgrade, please run: $ pip install wandb --upgrade

Tracking run with wandb version 0.15.3

Run data is saved locally in /Users/krystynamilian/Documents/praca/Cyfronet/epos/ai/repo/demo_scripts/notebooks/wandb/run-20230704_110544-8fry08nf

Syncing run fiery-sky-8 to Weights & Biases (docs)

View project at https://wandb.ai/epos/demo_scripts-notebooks

View run at https://wandb.ai/epos/demo_scripts-notebooks/runs/8fry08nf

wandb:   1 of 1 files downloaded.  

PhaseNet(
  (inc): Conv1d(3, 8, kernel_size=(7,), stride=(1,), padding=same)
  (in_bn): BatchNorm1d(8, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
  (down_branch): ModuleList(
    (0): ModuleList(
      (0): Conv1d(8, 8, kernel_size=(7,), stride=(1,), padding=same, bias=False)
      (1): BatchNorm1d(8, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (2): Conv1d(8, 8, kernel_size=(7,), stride=(4,), padding=(3,), bias=False)
      (3): BatchNorm1d(8, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
    )
    (1): ModuleList(
      (0): Conv1d(8, 16, kernel_size=(7,), stride=(1,), padding=same, bias=False)
      (1): BatchNorm1d(16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (2): Conv1d(16, 16, kernel_size=(7,), stride=(4,), bias=False)
      (3): BatchNorm1d(16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
    )
    (2): ModuleList(
      (0): Conv1d(16, 32, kernel_size=(7,), stride=(1,), padding=same, bias=False)
      (1): BatchNorm1d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (2): Conv1d(32, 32, kernel_size=(7,), stride=(4,), bias=False)
      (3): BatchNorm1d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
    )
    (3): ModuleList(
      (0): Conv1d(32, 64, kernel_size=(7,), stride=(1,), padding=same, bias=False)
      (1): BatchNorm1d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (2): Conv1d(64, 64, kernel_size=(7,), stride=(4,), bias=False)
      (3): BatchNorm1d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
    )
    (4): ModuleList(
      (0): Conv1d(64, 128, kernel_size=(7,), stride=(1,), padding=same, bias=False)
      (1): BatchNorm1d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (2-3): 2 x None
    )
  )
  (up_branch): ModuleList(
    (0): ModuleList(
      (0): ConvTranspose1d(128, 64, kernel_size=(7,), stride=(4,), bias=False)
      (1): BatchNorm1d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (2): Conv1d(128, 64, kernel_size=(7,), stride=(1,), padding=same, bias=False)
      (3): BatchNorm1d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
    )
    (1): ModuleList(
      (0): ConvTranspose1d(64, 32, kernel_size=(7,), stride=(4,), bias=False)
      (1): BatchNorm1d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (2): Conv1d(64, 32, kernel_size=(7,), stride=(1,), padding=same, bias=False)
      (3): BatchNorm1d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
    )
    (2): ModuleList(
      (0): ConvTranspose1d(32, 16, kernel_size=(7,), stride=(4,), bias=False)
      (1): BatchNorm1d(16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (2): Conv1d(32, 16, kernel_size=(7,), stride=(1,), padding=same, bias=False)
      (3): BatchNorm1d(16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
    )
    (3): ModuleList(
      (0): ConvTranspose1d(16, 8, kernel_size=(7,), stride=(4,), bias=False)
      (1): BatchNorm1d(8, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
      (2): Conv1d(16, 8, kernel_size=(7,), stride=(1,), padding=same, bias=False)
      (3): BatchNorm1d(8, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)
    )
  )
  (out): Conv1d(8, 2, kernel_size=(1,), stride=(1,), padding=same)
  (softmax): Softmax(dim=1)
)

In [3]:

data_path = '../../../data/igf/seisbench_format'
sampling_rate = 100
data = sbd.WaveformDataset(data_path, sampling_rate=sampling_rate)
data.filter(data.metadata.trace_Pg_arrival_sample.notna())

pick_mae = train.PickMAE(sampling_rate)
splits = ['train', 'dev', 'test']

In [4]:

all_samples = len(data.train()) + len(data.dev()) + len(data.test())
print(f"All samples: {all_samples}")
print(f"Training examples: {len(data.train())} {len(data.train())/all_samples * 100:.1f}%" )
print(f"Development examples: {len(data.dev())} {len(data.dev())/all_samples * 100:.1f}%")
print(f"Test examples: {len(data.test())} {len(data.test())/all_samples * 100:.1f} %")

All samples: 18002
Training examples: 12444 69.1%
Development examples: 2773 15.4%
Test examples: 2785 15.5 %

Calculate overall model performance

In [5]:

for split in splits: 
    print(f"\n\nModel resutls for {split} set")

    gen = train.get_data_generator(split=split, station=None, sampling_rate=sampling_rate, path=data_path, window='random')

    data_loader = DataLoader(gen, batch_size=256, shuffle=False, num_workers=0,
                                    worker_init_fn=worker_seeding)
        
    test_loss, test_mae = train.test_one_epoch(model, data_loader, pick_mae, wandb_log=False)

    
    

Model resutls for train set
train (12444, 17) 100
using random window
Test avg loss: 0.025075
Test avg mae: 0.046459



Model resutls for dev set
dev (2773, 17) 100
using random window
Test avg loss: 0.025158
Test avg mae: 0.049454



Model resutls for test set
test (2785, 17) 100
using random window
Test avg loss: 0.025469
Test avg mae: 0.047190

Check # frames per station in each set

In [6]:

frames_per_station = []
for split in splits: 
    frames_per_station.append(data.get_split(split).metadata.groupby('station_code').count()['index'])
    
frames_per_station = pd.DataFrame(frames_per_station, index=splits).transpose()
frames_per_station.plot(kind='bar', figsize=(15,3), title='Frames per station')

Out [6]:

<Axes: title={'center': 'Frames per station'}, xlabel='station_code'>

Calculate MAE per station for train/dev/test set

In [7]:

stations = data.metadata.station_code.unique()

results = []

for split in splits: 
    split_results = {}
    print(split)
    for station in stations: 
        print(station)
        split_results[station] = {'mae':[], 'loss':[]}
        for i in range(10):
            gen = train.get_data_generator(split=split, station=station, sampling_rate=sampling_rate, path=data_path)
            data_loader = DataLoader(gen, batch_size=256, shuffle=False, num_workers=0,
                                    worker_init_fn=worker_seeding)
        
            test_loss, test_mae = None, None
            try: 
                test_loss, test_mae = train.test_one_epoch(model, data_loader, pick_mae, wandb_log=False)
                test_mae = float(test_mae)
        
            except Exception as e: 
                print(e)
        
            split_results[station]['mae'].append(test_mae)
            split_results[station]['loss'].append(test_loss)
    results.append(split_results)
        
    
    
    

train
BRDW
train (12444, 17) 100
using random window
Test avg loss: 0.029197
Test avg mae: 0.101062

train (12444, 17) 100
using random window
Test avg loss: 0.028921
Test avg mae: 0.080500

train (12444, 17) 100
using random window
Test avg loss: 0.029244
Test avg mae: 0.086313

train (12444, 17) 100
using random window
Test avg loss: 0.029136
Test avg mae: 0.099563

train (12444, 17) 100
using random window
Test avg loss: 0.029101
Test avg mae: 0.100688

train (12444, 17) 100
using random window
Test avg loss: 0.028700
Test avg mae: 0.079062

train (12444, 17) 100
using random window
Test avg loss: 0.029309
Test avg mae: 0.098625

train (12444, 17) 100
using random window
Test avg loss: 0.029184
Test avg mae: 0.084937

train (12444, 17) 100
using random window
Test avg loss: 0.029323
Test avg mae: 0.086125

train (12444, 17) 100
using random window
Test avg loss: 0.029393
Test avg mae: 0.103312

GROD
train (12444, 17) 100
using random window
Test avg loss: 0.025653
Test avg mae: 0.062547

train (12444, 17) 100
using random window
Test avg loss: 0.025550
Test avg mae: 0.056665

train (12444, 17) 100
using random window
Test avg loss: 0.025562
Test avg mae: 0.055145

train (12444, 17) 100
using random window
Test avg loss: 0.025711
Test avg mae: 0.059011

train (12444, 17) 100
using random window
Test avg loss: 0.025634
Test avg mae: 0.057455

train (12444, 17) 100
using random window
Test avg loss: 0.025523
Test avg mae: 0.054818

train (12444, 17) 100
using random window
Test avg loss: 0.025585
Test avg mae: 0.056393

train (12444, 17) 100
using random window
Test avg loss: 0.025612
Test avg mae: 0.055862

train (12444, 17) 100
using random window
Test avg loss: 0.025611
Test avg mae: 0.058877

train (12444, 17) 100
using random window
Test avg loss: 0.025614
Test avg mae: 0.058422

GUZI
train (12444, 17) 100
using random window
Test avg loss: 0.024391
Test avg mae: 0.035747

train (12444, 17) 100
using random window
Test avg loss: 0.024418
Test avg mae: 0.036143

train (12444, 17) 100
using random window
Test avg loss: 0.024409
Test avg mae: 0.035239

train (12444, 17) 100
using random window
Test avg loss: 0.024416
Test avg mae: 0.035553

train (12444, 17) 100
using random window
Test avg loss: 0.024445
Test avg mae: 0.035810

train (12444, 17) 100
using random window
Test avg loss: 0.024413
Test avg mae: 0.037195

train (12444, 17) 100
using random window
Test avg loss: 0.024443
Test avg mae: 0.035182

train (12444, 17) 100
using random window
Test avg loss: 0.024415
Test avg mae: 0.036012

train (12444, 17) 100
using random window
Test avg loss: 0.024411
Test avg mae: 0.036120

train (12444, 17) 100
using random window
Test avg loss: 0.024401
Test avg mae: 0.036362

JEDR
train (12444, 17) 100
using random window
Test avg loss: 0.024772
Test avg mae: 0.036505

train (12444, 17) 100
using random window
Test avg loss: 0.024748
Test avg mae: 0.034873

train (12444, 17) 100
using random window
Test avg loss: 0.024829
Test avg mae: 0.036079

train (12444, 17) 100
using random window
Test avg loss: 0.024805
Test avg mae: 0.035852

train (12444, 17) 100
using random window
Test avg loss: 0.024755
Test avg mae: 0.034627

train (12444, 17) 100
using random window
Test avg loss: 0.024716
Test avg mae: 0.035023

train (12444, 17) 100
using random window
Test avg loss: 0.024677
Test avg mae: 0.033938

train (12444, 17) 100
using random window
Test avg loss: 0.024704
Test avg mae: 0.033226

train (12444, 17) 100
using random window
Test avg loss: 0.024675
Test avg mae: 0.034878

train (12444, 17) 100
using random window
Test avg loss: 0.024956
Test avg mae: 0.038911

MOSK2
train (12444, 17) 100
using random window
Test avg loss: 0.024905
Test avg mae: 0.041251

train (12444, 17) 100
using random window
Test avg loss: 0.024831
Test avg mae: 0.040558

train (12444, 17) 100
using random window
Test avg loss: 0.024893
Test avg mae: 0.041713

train (12444, 17) 100
using random window
Test avg loss: 0.024854
Test avg mae: 0.039448

train (12444, 17) 100
using random window
Test avg loss: 0.024832
Test avg mae: 0.040370

train (12444, 17) 100
using random window
Test avg loss: 0.024837
Test avg mae: 0.040740

train (12444, 17) 100
using random window
Test avg loss: 0.024842
Test avg mae: 0.040198

train (12444, 17) 100
using random window
Test avg loss: 0.024900
Test avg mae: 0.040395

train (12444, 17) 100
using random window
Test avg loss: 0.024862
Test avg mae: 0.039904

train (12444, 17) 100
using random window
Test avg loss: 0.024800
Test avg mae: 0.040857

NWLU
train (12444, 17) 100
using random window
Test avg loss: 0.024434
Test avg mae: 0.034428

train (12444, 17) 100
using random window
Test avg loss: 0.024436
Test avg mae: 0.034107

train (12444, 17) 100
using random window
Test avg loss: 0.024587
Test avg mae: 0.034464

train (12444, 17) 100
using random window
Test avg loss: 0.024417
Test avg mae: 0.034171

train (12444, 17) 100
using random window
Test avg loss: 0.024523
Test avg mae: 0.033232

train (12444, 17) 100
using random window
Test avg loss: 0.024620
Test avg mae: 0.034845

train (12444, 17) 100
using random window
Test avg loss: 0.024388
Test avg mae: 0.033914

train (12444, 17) 100
using random window
Test avg loss: 0.024565
Test avg mae: 0.034201

train (12444, 17) 100
using random window
Test avg loss: 0.024580
Test avg mae: 0.034112

train (12444, 17) 100
using random window
Test avg loss: 0.024502
Test avg mae: 0.033727

PCHB
train (12444, 17) 100
using random window
Test avg loss: 0.024624
Test avg mae: 0.041649

train (12444, 17) 100
using random window
Test avg loss: 0.024515
Test avg mae: 0.039478

train (12444, 17) 100
using random window
Test avg loss: 0.024490
Test avg mae: 0.040745

train (12444, 17) 100
using random window
Test avg loss: 0.024513
Test avg mae: 0.039800

train (12444, 17) 100
using random window
Test avg loss: 0.024597
Test avg mae: 0.042827

train (12444, 17) 100
using random window
Test avg loss: 0.024525
Test avg mae: 0.039057

train (12444, 17) 100
using random window
Test avg loss: 0.024588
Test avg mae: 0.042966

train (12444, 17) 100
using random window
Test avg loss: 0.024519
Test avg mae: 0.039862

train (12444, 17) 100
using random window
Test avg loss: 0.024510
Test avg mae: 0.040924

train (12444, 17) 100
using random window
Test avg loss: 0.024571
Test avg mae: 0.040463

PPOL
train (12444, 17) 100
using random window
Test avg loss: 0.025924
Test avg mae: 0.061875

train (12444, 17) 100
using random window
Test avg loss: 0.026050
Test avg mae: 0.066563

train (12444, 17) 100
using random window
Test avg loss: 0.025983
Test avg mae: 0.067147

train (12444, 17) 100
using random window
Test avg loss: 0.025980
Test avg mae: 0.065153

train (12444, 17) 100
using random window
Test avg loss: 0.026014
Test avg mae: 0.061698

train (12444, 17) 100
using random window
Test avg loss: 0.026035
Test avg mae: 0.061963

train (12444, 17) 100
using random window
Test avg loss: 0.025978
Test avg mae: 0.063233

train (12444, 17) 100
using random window
Test avg loss: 0.025947
Test avg mae: 0.063183

train (12444, 17) 100
using random window
Test avg loss: 0.025983
Test avg mae: 0.065201

train (12444, 17) 100
using random window
Test avg loss: 0.026024
Test avg mae: 0.064024

RUDN
train (12444, 17) 100
using random window
Test avg loss: 0.025199
Test avg mae: 0.044886

train (12444, 17) 100
using random window
Test avg loss: 0.025190
Test avg mae: 0.045990

train (12444, 17) 100
using random window
Test avg loss: 0.025130
Test avg mae: 0.045036

train (12444, 17) 100
using random window
Test avg loss: 0.025101
Test avg mae: 0.043167

train (12444, 17) 100
using random window
Test avg loss: 0.025226
Test avg mae: 0.047723

train (12444, 17) 100
using random window
Test avg loss: 0.025385
Test avg mae: 0.061125

train (12444, 17) 100
using random window
Test avg loss: 0.025306
Test avg mae: 0.046457

train (12444, 17) 100
using random window
Test avg loss: 0.025231
Test avg mae: 0.047450

train (12444, 17) 100
using random window
Test avg loss: 0.025327
Test avg mae: 0.047645

train (12444, 17) 100
using random window
Test avg loss: 0.025388
Test avg mae: 0.047236

RYNR
train (12444, 17) 100
using random window
Test avg loss: 0.025864
Test avg mae: 0.066633

train (12444, 17) 100
using random window
Test avg loss: 0.025840
Test avg mae: 0.065376

train (12444, 17) 100
using random window
Test avg loss: 0.025626
Test avg mae: 0.062124

train (12444, 17) 100
using random window
Test avg loss: 0.025880
Test avg mae: 0.066312

train (12444, 17) 100
using random window
Test avg loss: 0.025988
Test avg mae: 0.067691

train (12444, 17) 100
using random window
Test avg loss: 0.025817
Test avg mae: 0.067727

train (12444, 17) 100
using random window
Test avg loss: 0.025806
Test avg mae: 0.065661

train (12444, 17) 100
using random window
Test avg loss: 0.025949
Test avg mae: 0.063175

train (12444, 17) 100
using random window
Test avg loss: 0.025792
Test avg mae: 0.066198

train (12444, 17) 100
using random window
Test avg loss: 0.025799
Test avg mae: 0.066651

RZEC
train (12444, 17) 100
using random window
Test avg loss: 0.023818
Test avg mae: 0.031034

train (12444, 17) 100
using random window
Test avg loss: 0.023787
Test avg mae: 0.028276

train (12444, 17) 100
using random window
Test avg loss: 0.023800
Test avg mae: 0.024483

train (12444, 17) 100
using random window
Test avg loss: 0.023806
Test avg mae: 0.025862

train (12444, 17) 100
using random window
Test avg loss: 0.023858
Test avg mae: 0.028966

train (12444, 17) 100
using random window
Test avg loss: 0.023822
Test avg mae: 0.024828

train (12444, 17) 100
using random window
Test avg loss: 0.023817
Test avg mae: 0.027586

train (12444, 17) 100
using random window
Test avg loss: 0.023799
Test avg mae: 0.031379

train (12444, 17) 100
using random window
Test avg loss: 0.023909
Test avg mae: 0.031724

train (12444, 17) 100
using random window
Test avg loss: 0.023884
Test avg mae: 0.030345

SGOR
train (12444, 17) 100
using random window
Test avg loss: 0.024467
Test avg mae: 0.037522

train (12444, 17) 100
using random window
Test avg loss: 0.024469
Test avg mae: 0.036812

train (12444, 17) 100
using random window
Test avg loss: 0.024430
Test avg mae: 0.035673

train (12444, 17) 100
using random window
Test avg loss: 0.024438
Test avg mae: 0.034225

train (12444, 17) 100
using random window
Test avg loss: 0.024428
Test avg mae: 0.034473

train (12444, 17) 100
using random window
Test avg loss: 0.024397
Test avg mae: 0.036998

train (12444, 17) 100
using random window
Test avg loss: 0.024478
Test avg mae: 0.036346

train (12444, 17) 100
using random window
Test avg loss: 0.024488
Test avg mae: 0.035342

train (12444, 17) 100
using random window
Test avg loss: 0.024540
Test avg mae: 0.038125

train (12444, 17) 100
using random window
Test avg loss: 0.024425
Test avg mae: 0.035026

TRBC2
train (12444, 17) 100
using random window
Test avg loss: 0.024715
Test avg mae: 0.044793

train (12444, 17) 100
using random window
Test avg loss: 0.024758
Test avg mae: 0.046050

train (12444, 17) 100
using random window
Test avg loss: 0.024701
Test avg mae: 0.042903

train (12444, 17) 100
using random window
Test avg loss: 0.024757
Test avg mae: 0.045986

train (12444, 17) 100
using random window
Test avg loss: 0.024681
Test avg mae: 0.042041

train (12444, 17) 100
using random window
Test avg loss: 0.024607
Test avg mae: 0.044536

train (12444, 17) 100
using random window
Test avg loss: 0.024924
Test avg mae: 0.045319

train (12444, 17) 100
using random window
Test avg loss: 0.024695
Test avg mae: 0.044265

train (12444, 17) 100
using random window
Test avg loss: 0.024710
Test avg mae: 0.045154

train (12444, 17) 100
using random window
Test avg loss: 0.024677
Test avg mae: 0.045740

TRN2
train (12444, 17) 100
using random window
Test avg loss: 0.025530
Test avg mae: 0.054768

train (12444, 17) 100
using random window
Test avg loss: 0.025488
Test avg mae: 0.050913

train (12444, 17) 100
using random window
Test avg loss: 0.025473
Test avg mae: 0.054110

train (12444, 17) 100
using random window
Test avg loss: 0.025458
Test avg mae: 0.057963

train (12444, 17) 100
using random window
Test avg loss: 0.025466
Test avg mae: 0.054473

train (12444, 17) 100
using random window
Test avg loss: 0.025534
Test avg mae: 0.054216

train (12444, 17) 100
using random window
Test avg loss: 0.025420
Test avg mae: 0.049408

train (12444, 17) 100
using random window
Test avg loss: 0.025467
Test avg mae: 0.053733

train (12444, 17) 100
using random window
Test avg loss: 0.025430
Test avg mae: 0.051889

train (12444, 17) 100
using random window
Test avg loss: 0.025404
Test avg mae: 0.054043

TRZS
train (12444, 17) 100
using random window
Test avg loss: 0.024975
Test avg mae: 0.044880

train (12444, 17) 100
using random window
Test avg loss: 0.025067
Test avg mae: 0.043206

train (12444, 17) 100
using random window
Test avg loss: 0.025030
Test avg mae: 0.043493

train (12444, 17) 100
using random window
Test avg loss: 0.024915
Test avg mae: 0.044593

train (12444, 17) 100
using random window
Test avg loss: 0.025043
Test avg mae: 0.044641

train (12444, 17) 100
using random window
Test avg loss: 0.025157
Test avg mae: 0.045072

train (12444, 17) 100
using random window
Test avg loss: 0.024961
Test avg mae: 0.042823

train (12444, 17) 100
using random window
Test avg loss: 0.025094
Test avg mae: 0.044450

train (12444, 17) 100
using random window
Test avg loss: 0.025002
Test avg mae: 0.043589

train (12444, 17) 100
using random window
Test avg loss: 0.025029
Test avg mae: 0.044689

ZMST
train (12444, 17) 100
using random window
Test avg loss: 0.025080
Test avg mae: 0.049731

train (12444, 17) 100
using random window
Test avg loss: 0.025050
Test avg mae: 0.048796

train (12444, 17) 100
using random window
Test avg loss: 0.025179
Test avg mae: 0.048982

train (12444, 17) 100
using random window
Test avg loss: 0.025070
Test avg mae: 0.049723

train (12444, 17) 100
using random window
Test avg loss: 0.025061
Test avg mae: 0.048439

train (12444, 17) 100
using random window
Test avg loss: 0.025102
Test avg mae: 0.049314

train (12444, 17) 100
using random window
Test avg loss: 0.025127
Test avg mae: 0.050508

train (12444, 17) 100
using random window
Test avg loss: 0.025169
Test avg mae: 0.049843

train (12444, 17) 100
using random window
Test avg loss: 0.025097
Test avg mae: 0.051237

train (12444, 17) 100
using random window
Test avg loss: 0.025070
Test avg mae: 0.048802

LUBW
train (12444, 17) 100
using random window
Test avg loss: 0.028350
Test avg mae: 0.090000

train (12444, 17) 100
using random window
Test avg loss: 0.027360
Test avg mae: 0.064848

train (12444, 17) 100
using random window
Test avg loss: 0.027597
Test avg mae: 0.074545

train (12444, 17) 100
using random window
Test avg loss: 0.028823
Test avg mae: 0.088182

train (12444, 17) 100
using random window
Test avg loss: 0.028706
Test avg mae: 0.110000

train (12444, 17) 100
using random window
Test avg loss: 0.028709
Test avg mae: 0.093333

train (12444, 17) 100
using random window
Test avg loss: 0.028397
Test avg mae: 0.078182

train (12444, 17) 100
using random window
Test avg loss: 0.028619
Test avg mae: 0.095455

train (12444, 17) 100
using random window
Test avg loss: 0.028573
Test avg mae: 0.093333

train (12444, 17) 100
using random window
Test avg loss: 0.028804
Test avg mae: 0.119091

DWOL
train (12444, 17) 100
using random window
Test avg loss: 0.024134
Test avg mae: 0.027203

train (12444, 17) 100
using random window
Test avg loss: 0.024156
Test avg mae: 0.027302

train (12444, 17) 100
using random window
Test avg loss: 0.024178
Test avg mae: 0.027137

train (12444, 17) 100
using random window
Test avg loss: 0.024149
Test avg mae: 0.027665

train (12444, 17) 100
using random window
Test avg loss: 0.024316
Test avg mae: 0.036156

train (12444, 17) 100
using random window
Test avg loss: 0.024161
Test avg mae: 0.027512

train (12444, 17) 100
using random window
Test avg loss: 0.024177
Test avg mae: 0.035867

train (12444, 17) 100
using random window
Test avg loss: 0.024151
Test avg mae: 0.027882

train (12444, 17) 100
using random window
Test avg loss: 0.024269
Test avg mae: 0.035224

train (12444, 17) 100
using random window
Test avg loss: 0.024150
Test avg mae: 0.026653

LUBZ
train (12444, 17) 100
using random window
Test avg loss: 0.030519
Test avg mae: 0.160000

train (12444, 17) 100
using random window
Test avg loss: 0.032446
Test avg mae: 0.180000

train (12444, 17) 100
using random window
Test avg loss: 0.031204
Test avg mae: 0.160000

train (12444, 17) 100
using random window
Test avg loss: 0.031441
Test avg mae: 0.170000

train (12444, 17) 100
using random window
Test avg loss: 0.032243
Test avg mae: 0.180000

train (12444, 17) 100
using random window
Test avg loss: 0.033196
Test avg mae: 0.175000

train (12444, 17) 100
using random window
Test avg loss: 0.031445
Test avg mae: 0.170000

train (12444, 17) 100
using random window
Test avg loss: 0.031896
Test avg mae: 0.150000

train (12444, 17) 100
using random window
Test avg loss: 0.031475
Test avg mae: 0.160000

train (12444, 17) 100
using random window
Test avg loss: 0.031954
Test avg mae: 0.180000

ZUKW2
train (12444, 17) 100
using random window
Test avg loss: 0.024746
Test avg mae: 0.035442

train (12444, 17) 100
using random window
Test avg loss: 0.024587
Test avg mae: 0.033472

train (12444, 17) 100
using random window
Test avg loss: 0.024649
Test avg mae: 0.032509

train (12444, 17) 100
using random window
Test avg loss: 0.024574
Test avg mae: 0.033508

train (12444, 17) 100
using random window
Test avg loss: 0.024625
Test avg mae: 0.033792

train (12444, 17) 100
using random window
Test avg loss: 0.024838
Test avg mae: 0.034519

train (12444, 17) 100
using random window
Test avg loss: 0.024788
Test avg mae: 0.036248

train (12444, 17) 100
using random window
Test avg loss: 0.024626
Test avg mae: 0.033304

train (12444, 17) 100
using random window
Test avg loss: 0.024571
Test avg mae: 0.032324

train (12444, 17) 100
using random window
Test avg loss: 0.024572
Test avg mae: 0.031848

DABR
train (12444, 17) 100
using random window
Test avg loss: 0.024302
Test avg mae: 0.031127

train (12444, 17) 100
using random window
Test avg loss: 0.024232
Test avg mae: 0.061970

train (12444, 17) 100
using random window
Test avg loss: 0.024185
Test avg mae: 0.032058

train (12444, 17) 100
using random window
Test avg loss: 0.024129
Test avg mae: 0.031252

train (12444, 17) 100
using random window
Test avg loss: 0.024226
Test avg mae: 0.031566

train (12444, 17) 100
using random window
Test avg loss: 0.024113
Test avg mae: 0.030397

train (12444, 17) 100
using random window
Test avg loss: 0.024186
Test avg mae: 0.030980

train (12444, 17) 100
using random window
Test avg loss: 0.024138
Test avg mae: 0.031460

train (12444, 17) 100
using random window
Test avg loss: 0.024194
Test avg mae: 0.031061

train (12444, 17) 100
using random window
Test avg loss: 0.024163
Test avg mae: 0.031359

PEKW2
train (12444, 17) 100
using random window
Test avg loss: 0.024472
Test avg mae: 0.043610

train (12444, 17) 100
using random window
Test avg loss: 0.024489
Test avg mae: 0.044439

train (12444, 17) 100
using random window
Test avg loss: 0.024524
Test avg mae: 0.043707

train (12444, 17) 100
using random window
Test avg loss: 0.024507
Test avg mae: 0.042732

train (12444, 17) 100
using random window
Test avg loss: 0.024472
Test avg mae: 0.042488

train (12444, 17) 100
using random window
Test avg loss: 0.024517
Test avg mae: 0.042537

train (12444, 17) 100
using random window
Test avg loss: 0.024527
Test avg mae: 0.043707

train (12444, 17) 100
using random window
Test avg loss: 0.024483
Test avg mae: 0.042585

train (12444, 17) 100
using random window
Test avg loss: 0.024475
Test avg mae: 0.044293

train (12444, 17) 100
using random window
Test avg loss: 0.024497
Test avg mae: 0.041854

KRZY
train (12444, 17) 100
using random window
Test avg loss: 0.025522
Test avg mae: 0.054286

train (12444, 17) 100
using random window
Test avg loss: 0.025226
Test avg mae: 0.057143

train (12444, 17) 100
using random window
Test avg loss: 0.025248
Test avg mae: 0.070000

train (12444, 17) 100
using random window
Test avg loss: 0.025417
Test avg mae: 0.047143

train (12444, 17) 100
using random window
Test avg loss: 0.025459
Test avg mae: 0.051429

train (12444, 17) 100
using random window
Test avg loss: 0.025202
Test avg mae: 0.058571

train (12444, 17) 100
using random window
Test avg loss: 0.025341
Test avg mae: 0.060000

train (12444, 17) 100
using random window
Test avg loss: 0.025273
Test avg mae: 0.054286

train (12444, 17) 100
using random window
Test avg loss: 0.025228
Test avg mae: 0.058571

train (12444, 17) 100
using random window
Test avg loss: 0.025349
Test avg mae: 0.048571

OBIS
train (12444, 17) 100
using random window
Test avg loss: 0.023926
Test avg mae: 0.025517

train (12444, 17) 100
using random window
Test avg loss: 0.023971
Test avg mae: 0.026552

train (12444, 17) 100
using random window
Test avg loss: 0.023946
Test avg mae: 0.027793

train (12444, 17) 100
using random window
Test avg loss: 0.023967
Test avg mae: 0.024828

train (12444, 17) 100
using random window
Test avg loss: 0.023990
Test avg mae: 0.027586

train (12444, 17) 100
using random window
Test avg loss: 0.023950
Test avg mae: 0.026276

train (12444, 17) 100
using random window
Test avg loss: 0.023965
Test avg mae: 0.026345

train (12444, 17) 100
using random window
Test avg loss: 0.023990
Test avg mae: 0.025724

train (12444, 17) 100
using random window
Test avg loss: 0.023987
Test avg mae: 0.027034

train (12444, 17) 100
using random window
Test avg loss: 0.023955
Test avg mae: 0.027241

KAZI
train (12444, 17) 100
using random window
Test avg loss: 0.023951
Test avg mae: 0.027810

train (12444, 17) 100
using random window
Test avg loss: 0.023934
Test avg mae: 0.026762

train (12444, 17) 100
using random window
Test avg loss: 0.023937
Test avg mae: 0.027619

train (12444, 17) 100
using random window
Test avg loss: 0.024004
Test avg mae: 0.027429

train (12444, 17) 100
using random window
Test avg loss: 0.023971
Test avg mae: 0.025429

train (12444, 17) 100
using random window
Test avg loss: 0.023979
Test avg mae: 0.027810

train (12444, 17) 100
using random window
Test avg loss: 0.023944
Test avg mae: 0.026286

train (12444, 17) 100
using random window
Test avg loss: 0.023980
Test avg mae: 0.026762

train (12444, 17) 100
using random window
Test avg loss: 0.023900
Test avg mae: 0.026095

train (12444, 17) 100
using random window
Test avg loss: 0.023933
Test avg mae: 0.027048

KWLC
train (12444, 17) 100
using random window
division by zero
train (12444, 17) 100
using random window
division by zero
train (12444, 17) 100
using random window
division by zero
train (12444, 17) 100
using random window
division by zero
train (12444, 17) 100
using random window
division by zero
train (12444, 17) 100
using random window
division by zero
train (12444, 17) 100
using random window
division by zero
train (12444, 17) 100
using random window
division by zero
train (12444, 17) 100
using random window
division by zero
train (12444, 17) 100
using random window
division by zero
dev
BRDW
dev (2773, 17) 100
using random window
Test avg loss: 0.028971
Test avg mae: 0.086500

dev (2773, 17) 100
using random window
Test avg loss: 0.029175
Test avg mae: 0.086000

dev (2773, 17) 100
using random window
Test avg loss: 0.027710
Test avg mae: 0.085000

dev (2773, 17) 100
using random window
Test avg loss: 0.027695
Test avg mae: 0.084000

dev (2773, 17) 100
using random window
Test avg loss: 0.028312
Test avg mae: 0.087000

dev (2773, 17) 100
using random window
Test avg loss: 0.027334
Test avg mae: 0.083000

dev (2773, 17) 100
using random window
Test avg loss: 0.027557
Test avg mae: 0.088000

dev (2773, 17) 100
using random window
Test avg loss: 0.028396
Test avg mae: 0.093500

dev (2773, 17) 100
using random window
Test avg loss: 0.027768
Test avg mae: 0.085500

dev (2773, 17) 100
using random window
Test avg loss: 0.026457
Test avg mae: 0.089000

GROD
dev (2773, 17) 100
using random window
Test avg loss: 0.025171
Test avg mae: 0.048325

dev (2773, 17) 100
using random window
Test avg loss: 0.025280
Test avg mae: 0.048629

dev (2773, 17) 100
using random window
Test avg loss: 0.025265
Test avg mae: 0.050152

dev (2773, 17) 100
using random window
Test avg loss: 0.025124
Test avg mae: 0.044518

dev (2773, 17) 100
using random window
Test avg loss: 0.025599
Test avg mae: 0.056294

dev (2773, 17) 100
using random window
Test avg loss: 0.025292
Test avg mae: 0.046548

dev (2773, 17) 100
using random window
Test avg loss: 0.025199
Test avg mae: 0.048629

dev (2773, 17) 100
using random window
Test avg loss: 0.025107
Test avg mae: 0.048477

dev (2773, 17) 100
using random window
Test avg loss: 0.025218
Test avg mae: 0.048782

dev (2773, 17) 100
using random window
Test avg loss: 0.025200
Test avg mae: 0.050964

GUZI
dev (2773, 17) 100
using random window
Test avg loss: 0.024414
Test avg mae: 0.038017

dev (2773, 17) 100
using random window
Test avg loss: 0.024456
Test avg mae: 0.037025

dev (2773, 17) 100
using random window
Test avg loss: 0.024583
Test avg mae: 0.039917

dev (2773, 17) 100
using random window
Test avg loss: 0.024552
Test avg mae: 0.040496

dev (2773, 17) 100
using random window
Test avg loss: 0.024419
Test avg mae: 0.037190

dev (2773, 17) 100
using random window
Test avg loss: 0.024548
Test avg mae: 0.040992

dev (2773, 17) 100
using random window
Test avg loss: 0.024619
Test avg mae: 0.040413

dev (2773, 17) 100
using random window
Test avg loss: 0.024461
Test avg mae: 0.038843

dev (2773, 17) 100
using random window
Test avg loss: 0.024484
Test avg mae: 0.040248

dev (2773, 17) 100
using random window
Test avg loss: 0.024423
Test avg mae: 0.034628

JEDR
dev (2773, 17) 100
using random window
Test avg loss: 0.025750
Test avg mae: 0.008889

dev (2773, 17) 100
using random window
Test avg loss: 0.025724
Test avg mae: 0.012222

dev (2773, 17) 100
using random window
Test avg loss: 0.024942
Test avg mae: 0.015556

dev (2773, 17) 100
using random window
Test avg loss: 0.024687
Test avg mae: 0.013333

dev (2773, 17) 100
using random window
Test avg loss: 0.025583
Test avg mae: 0.016667

dev (2773, 17) 100
using random window
Test avg loss: 0.026014
Test avg mae: 0.018889

dev (2773, 17) 100
using random window
Test avg loss: 0.024612
Test avg mae: 0.015556

dev (2773, 17) 100
using random window
Test avg loss: 0.024265
Test avg mae: 0.013333

dev (2773, 17) 100
using random window
Test avg loss: 0.024791
Test avg mae: 0.008889

dev (2773, 17) 100
using random window
Test avg loss: 0.024083
Test avg mae: 0.006667

MOSK2
dev (2773, 17) 100
using random window
Test avg loss: 0.024982
Test avg mae: 0.043147

dev (2773, 17) 100
using random window
Test avg loss: 0.025065
Test avg mae: 0.039442

dev (2773, 17) 100
using random window
Test avg loss: 0.025059
Test avg mae: 0.045685

dev (2773, 17) 100
using random window
Test avg loss: 0.024984
Test avg mae: 0.046650

dev (2773, 17) 100
using random window
Test avg loss: 0.025198
Test avg mae: 0.053655

dev (2773, 17) 100
using random window
Test avg loss: 0.024918
Test avg mae: 0.049695

dev (2773, 17) 100
using random window
Test avg loss: 0.024984
Test avg mae: 0.044924

dev (2773, 17) 100
using random window
Test avg loss: 0.025159
Test avg mae: 0.039645

dev (2773, 17) 100
using random window
Test avg loss: 0.025128
Test avg mae: 0.045076

dev (2773, 17) 100
using random window
Test avg loss: 0.024987
Test avg mae: 0.045685

NWLU
dev (2773, 17) 100
using random window
Test avg loss: 0.025525
Test avg mae: 0.036667

dev (2773, 17) 100
using random window
Test avg loss: 0.024986
Test avg mae: 0.033913

dev (2773, 17) 100
using random window
Test avg loss: 0.024918
Test avg mae: 0.037609

dev (2773, 17) 100
using random window
Test avg loss: 0.025009
Test avg mae: 0.035507

dev (2773, 17) 100
using random window
Test avg loss: 0.024886
Test avg mae: 0.036304

dev (2773, 17) 100
using random window
Test avg loss: 0.025350
Test avg mae: 0.036159

dev (2773, 17) 100
using random window
Test avg loss: 0.025268
Test avg mae: 0.037174

dev (2773, 17) 100
using random window
Test avg loss: 0.025467
Test avg mae: 0.044203

dev (2773, 17) 100
using random window
Test avg loss: 0.025363
Test avg mae: 0.037101

dev (2773, 17) 100
using random window
Test avg loss: 0.025164
Test avg mae: 0.035072

PCHB
dev (2773, 17) 100
using random window
Test avg loss: 0.024568
Test avg mae: 0.041707

dev (2773, 17) 100
using random window
Test avg loss: 0.024671
Test avg mae: 0.043171

dev (2773, 17) 100
using random window
Test avg loss: 0.024638
Test avg mae: 0.040488

dev (2773, 17) 100
using random window
Test avg loss: 0.024669
Test avg mae: 0.043537

dev (2773, 17) 100
using random window
Test avg loss: 0.024710
Test avg mae: 0.042317

dev (2773, 17) 100
using random window
Test avg loss: 0.024639
Test avg mae: 0.039024

dev (2773, 17) 100
using random window
Test avg loss: 0.024660
Test avg mae: 0.042439

dev (2773, 17) 100
using random window
Test avg loss: 0.024627
Test avg mae: 0.041951

dev (2773, 17) 100
using random window
Test avg loss: 0.024666
Test avg mae: 0.040732

dev (2773, 17) 100
using random window
Test avg loss: 0.024709
Test avg mae: 0.044878

PPOL
dev (2773, 17) 100
using random window
Test avg loss: 0.025905
Test avg mae: 0.055522

dev (2773, 17) 100
using random window
Test avg loss: 0.025925
Test avg mae: 0.074627

dev (2773, 17) 100
using random window
Test avg loss: 0.026258
Test avg mae: 0.071493

dev (2773, 17) 100
using random window
Test avg loss: 0.025296
Test avg mae: 0.052836

dev (2773, 17) 100
using random window
Test avg loss: 0.025339
Test avg mae: 0.050000

dev (2773, 17) 100
using random window
Test avg loss: 0.026370
Test avg mae: 0.091791

dev (2773, 17) 100
using random window
Test avg loss: 0.025980
Test avg mae: 0.073881

dev (2773, 17) 100
using random window
Test avg loss: 0.026007
Test avg mae: 0.074328

dev (2773, 17) 100
using random window
Test avg loss: 0.025985
Test avg mae: 0.087910

dev (2773, 17) 100
using random window
Test avg loss: 0.025757
Test avg mae: 0.058507

RUDN
dev (2773, 17) 100
using random window
Test avg loss: 0.025465
Test avg mae: 0.047500

dev (2773, 17) 100
using random window
Test avg loss: 0.025528
Test avg mae: 0.049360

dev (2773, 17) 100
using random window
Test avg loss: 0.025330
Test avg mae: 0.048256

dev (2773, 17) 100
using random window
Test avg loss: 0.025654
Test avg mae: 0.050872

dev (2773, 17) 100
using random window
Test avg loss: 0.025423
Test avg mae: 0.049360

dev (2773, 17) 100
using random window
Test avg loss: 0.025523
Test avg mae: 0.051860

dev (2773, 17) 100
using random window
Test avg loss: 0.025558
Test avg mae: 0.047791

dev (2773, 17) 100
using random window
Test avg loss: 0.025529
Test avg mae: 0.048721

dev (2773, 17) 100
using random window
Test avg loss: 0.025503
Test avg mae: 0.050930

dev (2773, 17) 100
using random window
Test avg loss: 0.025489
Test avg mae: 0.047965

RYNR
dev (2773, 17) 100
using random window
Test avg loss: 0.026632
Test avg mae: 0.070596

dev (2773, 17) 100
using random window
Test avg loss: 0.026046
Test avg mae: 0.059868

dev (2773, 17) 100
using random window
Test avg loss: 0.026075
Test avg mae: 0.061192

dev (2773, 17) 100
using random window
Test avg loss: 0.026108
Test avg mae: 0.060728

dev (2773, 17) 100
using random window
Test avg loss: 0.026853
Test avg mae: 0.082980

dev (2773, 17) 100
using random window
Test avg loss: 0.026386
Test avg mae: 0.061523

dev (2773, 17) 100
using random window
Test avg loss: 0.027056
Test avg mae: 0.089735

dev (2773, 17) 100
using random window
Test avg loss: 0.026100
Test avg mae: 0.068874

dev (2773, 17) 100
using random window
Test avg loss: 0.026171
Test avg mae: 0.069868

dev (2773, 17) 100
using random window
Test avg loss: 0.026221
Test avg mae: 0.063046

RZEC
dev (2773, 17) 100
using random window
division by zero
dev (2773, 17) 100
using random window
division by zero
dev (2773, 17) 100
using random window
division by zero
dev (2773, 17) 100
using random window
division by zero
dev (2773, 17) 100
using random window
division by zero
dev (2773, 17) 100
using random window
division by zero
dev (2773, 17) 100
using random window
division by zero
dev (2773, 17) 100
using random window
division by zero
dev (2773, 17) 100
using random window
division by zero
dev (2773, 17) 100
using random window
division by zero
SGOR
dev (2773, 17) 100
using random window
Test avg loss: 0.024412
Test avg mae: 0.029419

dev (2773, 17) 100
using random window
Test avg loss: 0.025008
Test avg mae: 0.084194

dev (2773, 17) 100
using random window
Test avg loss: 0.025264
Test avg mae: 0.085161

dev (2773, 17) 100
using random window
Test avg loss: 0.024350
Test avg mae: 0.031032

dev (2773, 17) 100
using random window
Test avg loss: 0.024544
Test avg mae: 0.083613

dev (2773, 17) 100
using random window
Test avg loss: 0.024492
Test avg mae: 0.083226

dev (2773, 17) 100
using random window
Test avg loss: 0.024589
Test avg mae: 0.084387

dev (2773, 17) 100
using random window
Test avg loss: 0.025363
Test avg mae: 0.084452

dev (2773, 17) 100
using random window
Test avg loss: 0.024272
Test avg mae: 0.031806

dev (2773, 17) 100
using random window
Test avg loss: 0.025422
Test avg mae: 0.083548

TRBC2
dev (2773, 17) 100
using random window
Test avg loss: 0.024320
Test avg mae: 0.031864

dev (2773, 17) 100
using random window
Test avg loss: 0.024446
Test avg mae: 0.031864

dev (2773, 17) 100
using random window
Test avg loss: 0.024385
Test avg mae: 0.031864

dev (2773, 17) 100
using random window
Test avg loss: 0.024495
Test avg mae: 0.034068

dev (2773, 17) 100
using random window
Test avg loss: 0.024545
Test avg mae: 0.032712

dev (2773, 17) 100
using random window
Test avg loss: 0.024661
Test avg mae: 0.034746

dev (2773, 17) 100
using random window
Test avg loss: 0.024446
Test avg mae: 0.030339

dev (2773, 17) 100
using random window
Test avg loss: 0.024298
Test avg mae: 0.031356

dev (2773, 17) 100
using random window
Test avg loss: 0.024805
Test avg mae: 0.033390

dev (2773, 17) 100
using random window
Test avg loss: 0.024428
Test avg mae: 0.033390

TRN2
dev (2773, 17) 100
using random window
Test avg loss: 0.024707
Test avg mae: 0.039503

dev (2773, 17) 100
using random window
Test avg loss: 0.024751
Test avg mae: 0.040442

dev (2773, 17) 100
using random window
Test avg loss: 0.024843
Test avg mae: 0.039613

dev (2773, 17) 100
using random window
Test avg loss: 0.024716
Test avg mae: 0.041713

dev (2773, 17) 100
using random window
Test avg loss: 0.024761
Test avg mae: 0.039171

dev (2773, 17) 100
using random window
Test avg loss: 0.024794
Test avg mae: 0.041160

dev (2773, 17) 100
using random window
Test avg loss: 0.024743
Test avg mae: 0.040497

dev (2773, 17) 100
using random window
Test avg loss: 0.024749
Test avg mae: 0.040994

dev (2773, 17) 100
using random window
Test avg loss: 0.024759
Test avg mae: 0.040166

dev (2773, 17) 100
using random window
Test avg loss: 0.024745
Test avg mae: 0.040331

TRZS
dev (2773, 17) 100
using random window
Test avg loss: 0.026324
Test avg mae: 0.089741

dev (2773, 17) 100
using random window
Test avg loss: 0.025970
Test avg mae: 0.041897

dev (2773, 17) 100
using random window
Test avg loss: 0.026548
Test avg mae: 0.179310

dev (2773, 17) 100
using random window
Test avg loss: 0.026197
Test avg mae: 0.038534

dev (2773, 17) 100
using random window
Test avg loss: 0.026940
Test avg mae: 0.093362

dev (2773, 17) 100
using random window
Test avg loss: 0.026428
Test avg mae: 0.177241

dev (2773, 17) 100
using random window
Test avg loss: 0.026037
Test avg mae: 0.177414

dev (2773, 17) 100
using random window
Test avg loss: 0.026034
Test avg mae: 0.090948

dev (2773, 17) 100
using random window
Test avg loss: 0.026646
Test avg mae: 0.176897

dev (2773, 17) 100
using random window
Test avg loss: 0.026593
Test avg mae: 0.090776

ZMST
dev (2773, 17) 100
using random window
Test avg loss: 0.024322
Test avg mae: 0.029791

dev (2773, 17) 100
using random window
Test avg loss: 0.024131
Test avg mae: 0.028063

dev (2773, 17) 100
using random window
Test avg loss: 0.024258
Test avg mae: 0.029215

dev (2773, 17) 100
using random window
Test avg loss: 0.024214
Test avg mae: 0.030157

dev (2773, 17) 100
using random window
Test avg loss: 0.024274
Test avg mae: 0.031518

dev (2773, 17) 100
using random window
Test avg loss: 0.024378
Test avg mae: 0.034660

dev (2773, 17) 100
using random window
Test avg loss: 0.024129
Test avg mae: 0.027958

dev (2773, 17) 100
using random window
Test avg loss: 0.024210
Test avg mae: 0.029581

dev (2773, 17) 100
using random window
Test avg loss: 0.024291
Test avg mae: 0.034136

dev (2773, 17) 100
using random window
Test avg loss: 0.024350
Test avg mae: 0.030052

LUBW
dev (2773, 17) 100
using random window
Test avg loss: 0.029929
Test avg mae: 0.117917

dev (2773, 17) 100
using random window
Test avg loss: 0.029545
Test avg mae: 0.113333

dev (2773, 17) 100
using random window
Test avg loss: 0.030242
Test avg mae: 0.134583

dev (2773, 17) 100
using random window
Test avg loss: 0.028370
Test avg mae: 0.089167

dev (2773, 17) 100
using random window
Test avg loss: 0.028703
Test avg mae: 0.119583

dev (2773, 17) 100
using random window
Test avg loss: 0.030046
Test avg mae: 0.122083

dev (2773, 17) 100
using random window
Test avg loss: 0.030243
Test avg mae: 0.130833

dev (2773, 17) 100
using random window
Test avg loss: 0.030052
Test avg mae: 0.123750

dev (2773, 17) 100
using random window
Test avg loss: 0.027626
Test avg mae: 0.077917

dev (2773, 17) 100
using random window
Test avg loss: 0.028850
Test avg mae: 0.120833

DWOL
dev (2773, 17) 100
using random window
Test avg loss: 0.024480
Test avg mae: 0.048835

dev (2773, 17) 100
using random window
Test avg loss: 0.024443
Test avg mae: 0.031650

dev (2773, 17) 100
using random window
Test avg loss: 0.024452
Test avg mae: 0.033495

dev (2773, 17) 100
using random window
Test avg loss: 0.024461
Test avg mae: 0.032136

dev (2773, 17) 100
using random window
Test avg loss: 0.024591
Test avg mae: 0.048641

dev (2773, 17) 100
using random window
Test avg loss: 0.024491
Test avg mae: 0.032282

dev (2773, 17) 100
using random window
Test avg loss: 0.024380
Test avg mae: 0.032476

dev (2773, 17) 100
using random window
Test avg loss: 0.024482
Test avg mae: 0.049563

dev (2773, 17) 100
using random window
Test avg loss: 0.024437
Test avg mae: 0.032913

dev (2773, 17) 100
using random window
Test avg loss: 0.024607
Test avg mae: 0.049029

LUBZ
dev (2773, 17) 100
using random window
Test avg loss: 0.023679
Test avg mae: 0.020000

dev (2773, 17) 100
using random window
Test avg loss: 0.023553
Test avg mae: 0.006667

dev (2773, 17) 100
using random window
Test avg loss: 0.023672
Test avg mae: 0.026667

dev (2773, 17) 100
using random window
Test avg loss: 0.023809
Test avg mae: 0.013333

dev (2773, 17) 100
using random window
Test avg loss: 0.023596
Test avg mae: 0.006667

dev (2773, 17) 100
using random window
Test avg loss: 0.023617
Test avg mae: 0.010000

dev (2773, 17) 100
using random window
Test avg loss: 0.023756
Test avg mae: 0.020000

dev (2773, 17) 100
using random window
Test avg loss: 0.023778
Test avg mae: 0.016667

dev (2773, 17) 100
using random window
Test avg loss: 0.023575
Test avg mae: 0.010000

dev (2773, 17) 100
using random window
Test avg loss: 0.023789
Test avg mae: 0.020000

ZUKW2
dev (2773, 17) 100
using random window
Test avg loss: 0.024717
Test avg mae: 0.036456

dev (2773, 17) 100
using random window
Test avg loss: 0.024627
Test avg mae: 0.035534

dev (2773, 17) 100
using random window
Test avg loss: 0.024642
Test avg mae: 0.035194

dev (2773, 17) 100
using random window
Test avg loss: 0.024675
Test avg mae: 0.038835

dev (2773, 17) 100
using random window
Test avg loss: 0.024625
Test avg mae: 0.036359

dev (2773, 17) 100
using random window
Test avg loss: 0.024703
Test avg mae: 0.037282

dev (2773, 17) 100
using random window
Test avg loss: 0.024708
Test avg mae: 0.037961

dev (2773, 17) 100
using random window
Test avg loss: 0.024797
Test avg mae: 0.036650

dev (2773, 17) 100
using random window
Test avg loss: 0.024798
Test avg mae: 0.053786

dev (2773, 17) 100
using random window
Test avg loss: 0.024559
Test avg mae: 0.036650

DABR
dev (2773, 17) 100
using random window
Test avg loss: 0.024410
Test avg mae: 0.036832

dev (2773, 17) 100
using random window
Test avg loss: 0.024548
Test avg mae: 0.036460

dev (2773, 17) 100
using random window
Test avg loss: 0.024549
Test avg mae: 0.034596

dev (2773, 17) 100
using random window
Test avg loss: 0.024566
Test avg mae: 0.036708

dev (2773, 17) 100
using random window
Test avg loss: 0.024553
Test avg mae: 0.036273

dev (2773, 17) 100
using random window
Test avg loss: 0.024526
Test avg mae: 0.037143

dev (2773, 17) 100
using random window
Test avg loss: 0.024570
Test avg mae: 0.037143

dev (2773, 17) 100
using random window
Test avg loss: 0.024582
Test avg mae: 0.036708

dev (2773, 17) 100
using random window
Test avg loss: 0.024457
Test avg mae: 0.036708

dev (2773, 17) 100
using random window
Test avg loss: 0.024530
Test avg mae: 0.036584

PEKW2
dev (2773, 17) 100
using random window
Test avg loss: 0.026632
Test avg mae: 0.045361

dev (2773, 17) 100
using random window
Test avg loss: 0.026724
Test avg mae: 0.047938

dev (2773, 17) 100
using random window
Test avg loss: 0.026484
Test avg mae: 0.046392

dev (2773, 17) 100
using random window
Test avg loss: 0.026670
Test avg mae: 0.048247

dev (2773, 17) 100
using random window
Test avg loss: 0.026608
Test avg mae: 0.045464

dev (2773, 17) 100
using random window
Test avg loss: 0.026605
Test avg mae: 0.046804

dev (2773, 17) 100
using random window
Test avg loss: 0.026637
Test avg mae: 0.047629

dev (2773, 17) 100
using random window
Test avg loss: 0.026723
Test avg mae: 0.046804

dev (2773, 17) 100
using random window
Test avg loss: 0.026733
Test avg mae: 0.047010

dev (2773, 17) 100
using random window
Test avg loss: 0.026889
Test avg mae: 0.058454

KRZY
dev (2773, 17) 100
using random window
Test avg loss: 0.038278
Test avg mae: 0.110000

dev (2773, 17) 100
using random window
Test avg loss: 0.038771
Test avg mae: 0.097273

dev (2773, 17) 100
using random window
Test avg loss: 0.038392
Test avg mae: 0.099091

dev (2773, 17) 100
using random window
Test avg loss: 0.038522
Test avg mae: 0.104545

dev (2773, 17) 100
using random window
Test avg loss: 0.038693
Test avg mae: 0.104545

dev (2773, 17) 100
using random window
Test avg loss: 0.038756
Test avg mae: 0.096364

dev (2773, 17) 100
using random window
Test avg loss: 0.038586
Test avg mae: 0.099091

dev (2773, 17) 100
using random window
Test avg loss: 0.038925
Test avg mae: 0.103636

dev (2773, 17) 100
using random window
Test avg loss: 0.038687
Test avg mae: 0.098182

dev (2773, 17) 100
using random window
Test avg loss: 0.038287
Test avg mae: 0.106364

OBIS
dev (2773, 17) 100
using random window
Test avg loss: 0.024280
Test avg mae: 0.027320

dev (2773, 17) 100
using random window
Test avg loss: 0.024778
Test avg mae: 0.115155

dev (2773, 17) 100
using random window
Test avg loss: 0.024274
Test avg mae: 0.027938

dev (2773, 17) 100
using random window
Test avg loss: 0.024968
Test avg mae: 0.113505

dev (2773, 17) 100
using random window
Test avg loss: 0.024193
Test avg mae: 0.027938

dev (2773, 17) 100
using random window
Test avg loss: 0.024322
Test avg mae: 0.027526

dev (2773, 17) 100
using random window
Test avg loss: 0.024552
Test avg mae: 0.113608

dev (2773, 17) 100
using random window
Test avg loss: 0.024237
Test avg mae: 0.028041

dev (2773, 17) 100
using random window
Test avg loss: 0.024345
Test avg mae: 0.027732

dev (2773, 17) 100
using random window
Test avg loss: 0.024254
Test avg mae: 0.028763

KAZI
dev (2773, 17) 100
using random window
Test avg loss: 0.024294
Test avg mae: 0.029000

dev (2773, 17) 100
using random window
Test avg loss: 0.024415
Test avg mae: 0.029909

dev (2773, 17) 100
using random window
Test avg loss: 0.024394
Test avg mae: 0.030636

dev (2773, 17) 100
using random window
Test avg loss: 0.024499
Test avg mae: 0.028727

dev (2773, 17) 100
using random window
Test avg loss: 0.024466
Test avg mae: 0.030636

dev (2773, 17) 100
using random window
Test avg loss: 0.024289
Test avg mae: 0.029364

dev (2773, 17) 100
using random window
Test avg loss: 0.024452
Test avg mae: 0.028545

dev (2773, 17) 100
using random window
Test avg loss: 0.024398
Test avg mae: 0.030091

dev (2773, 17) 100
using random window
Test avg loss: 0.024537
Test avg mae: 0.029364

dev (2773, 17) 100
using random window
Test avg loss: 0.024418
Test avg mae: 0.031091

KWLC
dev (2773, 17) 100
using random window
Test avg loss: 0.023724
Test avg mae: 0.030000

dev (2773, 17) 100
using random window
Test avg loss: 0.023606
Test avg mae: 0.015000

dev (2773, 17) 100
using random window
Test avg loss: 0.023589
Test avg mae: 0.010000

dev (2773, 17) 100
using random window
Test avg loss: 0.023717
Test avg mae: 0.020000

dev (2773, 17) 100
using random window
Test avg loss: 0.023802
Test avg mae: 0.025000

dev (2773, 17) 100
using random window
Test avg loss: 0.023751
Test avg mae: 0.025000

dev (2773, 17) 100
using random window
Test avg loss: 0.023677
Test avg mae: 0.010000

dev (2773, 17) 100
using random window
Test avg loss: 0.024050
Test avg mae: 0.040000

dev (2773, 17) 100
using random window
Test avg loss: 0.023581
Test avg mae: 0.020000

dev (2773, 17) 100
using random window
Test avg loss: 0.023595
Test avg mae: 0.015000

test
BRDW
test (2785, 17) 100
using random window
Test avg loss: 0.025934
Test avg mae: 0.060896

test (2785, 17) 100
using random window
Test avg loss: 0.026379
Test avg mae: 0.073284

test (2785, 17) 100
using random window
Test avg loss: 0.026053
Test avg mae: 0.069552

test (2785, 17) 100
using random window
Test avg loss: 0.026029
Test avg mae: 0.060448

test (2785, 17) 100
using random window
Test avg loss: 0.026420
Test avg mae: 0.058358

test (2785, 17) 100
using random window
Test avg loss: 0.026025
Test avg mae: 0.064627

test (2785, 17) 100
using random window
Test avg loss: 0.025910
Test avg mae: 0.060448

test (2785, 17) 100
using random window
Test avg loss: 0.026421
Test avg mae: 0.086119

test (2785, 17) 100
using random window
Test avg loss: 0.026229
Test avg mae: 0.059552

test (2785, 17) 100
using random window
Test avg loss: 0.026069
Test avg mae: 0.061045

GROD
test (2785, 17) 100
using random window
Test avg loss: 0.025325
Test avg mae: 0.044286

test (2785, 17) 100
using random window
Test avg loss: 0.025248
Test avg mae: 0.044286

test (2785, 17) 100
using random window
Test avg loss: 0.025428
Test avg mae: 0.047029

test (2785, 17) 100
using random window
Test avg loss: 0.025281
Test avg mae: 0.042686

test (2785, 17) 100
using random window
Test avg loss: 0.025347
Test avg mae: 0.047200

test (2785, 17) 100
using random window
Test avg loss: 0.025475
Test avg mae: 0.048343

test (2785, 17) 100
using random window
Test avg loss: 0.025273
Test avg mae: 0.060286

test (2785, 17) 100
using random window
Test avg loss: 0.025299
Test avg mae: 0.046743

test (2785, 17) 100
using random window
Test avg loss: 0.025243
Test avg mae: 0.045829

test (2785, 17) 100
using random window
Test avg loss: 0.025070
Test avg mae: 0.043600

GUZI
test (2785, 17) 100
using random window
Test avg loss: 0.025129
Test avg mae: 0.040556

test (2785, 17) 100
using random window
Test avg loss: 0.025515
Test avg mae: 0.050556

test (2785, 17) 100
using random window
Test avg loss: 0.025496
Test avg mae: 0.049841

test (2785, 17) 100
using random window
Test avg loss: 0.025842
Test avg mae: 0.050238

test (2785, 17) 100
using random window
Test avg loss: 0.025786
Test avg mae: 0.058254

test (2785, 17) 100
using random window
Test avg loss: 0.025247
Test avg mae: 0.042222

test (2785, 17) 100
using random window
Test avg loss: 0.025569
Test avg mae: 0.049841

test (2785, 17) 100
using random window
Test avg loss: 0.025710
Test avg mae: 0.049048

test (2785, 17) 100
using random window
Test avg loss: 0.025632
Test avg mae: 0.050476

test (2785, 17) 100
using random window
Test avg loss: 0.025607
Test avg mae: 0.059206

JEDR
test (2785, 17) 100
using random window
Test avg loss: 0.027847
Test avg mae: 0.201290

test (2785, 17) 100
using random window
Test avg loss: 0.026951
Test avg mae: 0.043387

test (2785, 17) 100
using random window
Test avg loss: 0.027874
Test avg mae: 0.197258

test (2785, 17) 100
using random window
Test avg loss: 0.027162
Test avg mae: 0.042581

test (2785, 17) 100
using random window
Test avg loss: 0.026982
Test avg mae: 0.040161

test (2785, 17) 100
using random window
Test avg loss: 0.027034
Test avg mae: 0.042097

test (2785, 17) 100
using random window
Test avg loss: 0.027053
Test avg mae: 0.043710

test (2785, 17) 100
using random window
Test avg loss: 0.027056
Test avg mae: 0.041774

test (2785, 17) 100
using random window
Test avg loss: 0.027195
Test avg mae: 0.042903

test (2785, 17) 100
using random window
Test avg loss: 0.026985
Test avg mae: 0.043548

MOSK2
test (2785, 17) 100
using random window
Test avg loss: 0.024563
Test avg mae: 0.036432

test (2785, 17) 100
using random window
Test avg loss: 0.024515
Test avg mae: 0.035276

test (2785, 17) 100
using random window
Test avg loss: 0.024500
Test avg mae: 0.035276

test (2785, 17) 100
using random window
Test avg loss: 0.024548
Test avg mae: 0.034724

test (2785, 17) 100
using random window
Test avg loss: 0.024531
Test avg mae: 0.035678

test (2785, 17) 100
using random window
Test avg loss: 0.024597
Test avg mae: 0.035779

test (2785, 17) 100
using random window
Test avg loss: 0.024430
Test avg mae: 0.035879

test (2785, 17) 100
using random window
Test avg loss: 0.024420
Test avg mae: 0.033618

test (2785, 17) 100
using random window
Test avg loss: 0.024539
Test avg mae: 0.035377

test (2785, 17) 100
using random window
Test avg loss: 0.024518
Test avg mae: 0.035327

NWLU
test (2785, 17) 100
using random window
Test avg loss: 0.024630
Test avg mae: 0.031074

test (2785, 17) 100
using random window
Test avg loss: 0.024590
Test avg mae: 0.030872

test (2785, 17) 100
using random window
Test avg loss: 0.024608
Test avg mae: 0.030268

test (2785, 17) 100
using random window
Test avg loss: 0.024637
Test avg mae: 0.028591

test (2785, 17) 100
using random window
Test avg loss: 0.024670
Test avg mae: 0.029866

test (2785, 17) 100
using random window
Test avg loss: 0.024449
Test avg mae: 0.029597

test (2785, 17) 100
using random window
Test avg loss: 0.024694
Test avg mae: 0.030201

test (2785, 17) 100
using random window
Test avg loss: 0.024591
Test avg mae: 0.031544

test (2785, 17) 100
using random window
Test avg loss: 0.024664
Test avg mae: 0.029396

test (2785, 17) 100
using random window
Test avg loss: 0.024568
Test avg mae: 0.030470

PCHB
test (2785, 17) 100
using random window
Test avg loss: 0.024545
Test avg mae: 0.037215

test (2785, 17) 100
using random window
Test avg loss: 0.024679
Test avg mae: 0.035443

test (2785, 17) 100
using random window
Test avg loss: 0.024926
Test avg mae: 0.045190

test (2785, 17) 100
using random window
Test avg loss: 0.024639
Test avg mae: 0.035696

test (2785, 17) 100
using random window
Test avg loss: 0.024640
Test avg mae: 0.036582

test (2785, 17) 100
using random window
Test avg loss: 0.024553
Test avg mae: 0.033797

test (2785, 17) 100
using random window
Test avg loss: 0.024715
Test avg mae: 0.044557

test (2785, 17) 100
using random window
Test avg loss: 0.024856
Test avg mae: 0.045949

test (2785, 17) 100
using random window
Test avg loss: 0.024775
Test avg mae: 0.046076

test (2785, 17) 100
using random window
Test avg loss: 0.024738
Test avg mae: 0.035570

PPOL
test (2785, 17) 100
using random window
Test avg loss: 0.025437
Test avg mae: 0.048475

test (2785, 17) 100
using random window
Test avg loss: 0.025318
Test avg mae: 0.050000

test (2785, 17) 100
using random window
Test avg loss: 0.025423
Test avg mae: 0.049661

test (2785, 17) 100
using random window
Test avg loss: 0.025431
Test avg mae: 0.049153

test (2785, 17) 100
using random window
Test avg loss: 0.025138
Test avg mae: 0.048305

test (2785, 17) 100
using random window
Test avg loss: 0.025164
Test avg mae: 0.048475

test (2785, 17) 100
using random window
Test avg loss: 0.025158
Test avg mae: 0.041695

test (2785, 17) 100
using random window
Test avg loss: 0.025334
Test avg mae: 0.050847

test (2785, 17) 100
using random window
Test avg loss: 0.025233
Test avg mae: 0.048983

test (2785, 17) 100
using random window
Test avg loss: 0.025176
Test avg mae: 0.048644

RUDN
test (2785, 17) 100
using random window
Test avg loss: 0.025884
Test avg mae: 0.054605

test (2785, 17) 100
using random window
Test avg loss: 0.025868
Test avg mae: 0.059211

test (2785, 17) 100
using random window
Test avg loss: 0.026080
Test avg mae: 0.057566

test (2785, 17) 100
using random window
Test avg loss: 0.025783
Test avg mae: 0.058158

test (2785, 17) 100
using random window
Test avg loss: 0.025573
Test avg mae: 0.044605

test (2785, 17) 100
using random window
Test avg loss: 0.026530
Test avg mae: 0.055329

test (2785, 17) 100
using random window
Test avg loss: 0.026172
Test avg mae: 0.057434

test (2785, 17) 100
using random window
Test avg loss: 0.026277
Test avg mae: 0.057303

test (2785, 17) 100
using random window
Test avg loss: 0.026563
Test avg mae: 0.058487

test (2785, 17) 100
using random window
Test avg loss: 0.026182
Test avg mae: 0.055592

RYNR
test (2785, 17) 100
using random window
Test avg loss: 0.027648
Test avg mae: 0.086963

test (2785, 17) 100
using random window
Test avg loss: 0.027588
Test avg mae: 0.082815

test (2785, 17) 100
using random window
Test avg loss: 0.027658
Test avg mae: 0.089778

test (2785, 17) 100
using random window
Test avg loss: 0.027625
Test avg mae: 0.108296

test (2785, 17) 100
using random window
Test avg loss: 0.026857
Test avg mae: 0.079926

test (2785, 17) 100
using random window
Test avg loss: 0.027323
Test avg mae: 0.098370

test (2785, 17) 100
using random window
Test avg loss: 0.026899
Test avg mae: 0.076296

test (2785, 17) 100
using random window
Test avg loss: 0.027445
Test avg mae: 0.077481

test (2785, 17) 100
using random window
Test avg loss: 0.027287
Test avg mae: 0.086963

test (2785, 17) 100
using random window
Test avg loss: 0.026892
Test avg mae: 0.084148

RZEC
test (2785, 17) 100
using random window
division by zero
test (2785, 17) 100
using random window
division by zero
test (2785, 17) 100
using random window
division by zero
test (2785, 17) 100
using random window
division by zero
test (2785, 17) 100
using random window
division by zero
test (2785, 17) 100
using random window
division by zero
test (2785, 17) 100
using random window
division by zero
test (2785, 17) 100
using random window
division by zero
test (2785, 17) 100
using random window
division by zero
test (2785, 17) 100
using random window
division by zero
SGOR
test (2785, 17) 100
using random window
Test avg loss: 0.025765
Test avg mae: 0.042595

test (2785, 17) 100
using random window
Test avg loss: 0.025846
Test avg mae: 0.042848

test (2785, 17) 100
using random window
Test avg loss: 0.025771
Test avg mae: 0.043861

test (2785, 17) 100
using random window
Test avg loss: 0.026033
Test avg mae: 0.042532

test (2785, 17) 100
using random window
Test avg loss: 0.025628
Test avg mae: 0.043734

test (2785, 17) 100
using random window
Test avg loss: 0.025968
Test avg mae: 0.043987

test (2785, 17) 100
using random window
Test avg loss: 0.025798
Test avg mae: 0.042848

test (2785, 17) 100
using random window
Test avg loss: 0.026101
Test avg mae: 0.045063

test (2785, 17) 100
using random window
Test avg loss: 0.025852
Test avg mae: 0.042468

test (2785, 17) 100
using random window
Test avg loss: 0.025887
Test avg mae: 0.044241

TRBC2
test (2785, 17) 100
using random window
Test avg loss: 0.024363
Test avg mae: 0.033091

test (2785, 17) 100
using random window
Test avg loss: 0.024329
Test avg mae: 0.034000

test (2785, 17) 100
using random window
Test avg loss: 0.024381
Test avg mae: 0.032000

test (2785, 17) 100
using random window
Test avg loss: 0.024374
Test avg mae: 0.032545

test (2785, 17) 100
using random window
Test avg loss: 0.024416
Test avg mae: 0.036182

test (2785, 17) 100
using random window
Test avg loss: 0.024293
Test avg mae: 0.031273

test (2785, 17) 100
using random window
Test avg loss: 0.024474
Test avg mae: 0.036727

test (2785, 17) 100
using random window
Test avg loss: 0.024390
Test avg mae: 0.036182

test (2785, 17) 100
using random window
Test avg loss: 0.024293
Test avg mae: 0.031818

test (2785, 17) 100
using random window
Test avg loss: 0.024265
Test avg mae: 0.033273

TRN2
test (2785, 17) 100
using random window
Test avg loss: 0.027654
Test avg mae: 0.063333

test (2785, 17) 100
using random window
Test avg loss: 0.027777
Test avg mae: 0.060864

test (2785, 17) 100
using random window
Test avg loss: 0.027985
Test avg mae: 0.072531

test (2785, 17) 100
using random window
Test avg loss: 0.028154
Test avg mae: 0.072840

test (2785, 17) 100
using random window
Test avg loss: 0.027799
Test avg mae: 0.070802

test (2785, 17) 100
using random window
Test avg loss: 0.027817
Test avg mae: 0.064198

test (2785, 17) 100
using random window
Test avg loss: 0.027730
Test avg mae: 0.064506

test (2785, 17) 100
using random window
Test avg loss: 0.027682
Test avg mae: 0.068889

test (2785, 17) 100
using random window
Test avg loss: 0.027775
Test avg mae: 0.064444

test (2785, 17) 100
using random window
Test avg loss: 0.027888
Test avg mae: 0.071111

TRZS
test (2785, 17) 100
using random window
Test avg loss: 0.024738
Test avg mae: 0.037113

test (2785, 17) 100
using random window
Test avg loss: 0.024839
Test avg mae: 0.036701

test (2785, 17) 100
using random window
Test avg loss: 0.024709
Test avg mae: 0.036082

test (2785, 17) 100
using random window
Test avg loss: 0.024755
Test avg mae: 0.035773

test (2785, 17) 100
using random window
Test avg loss: 0.024726
Test avg mae: 0.035567

test (2785, 17) 100
using random window
Test avg loss: 0.024823
Test avg mae: 0.036701

test (2785, 17) 100
using random window
Test avg loss: 0.024761
Test avg mae: 0.035052

test (2785, 17) 100
using random window
Test avg loss: 0.024710
Test avg mae: 0.033918

test (2785, 17) 100
using random window
Test avg loss: 0.024731
Test avg mae: 0.038454

test (2785, 17) 100
using random window
Test avg loss: 0.024848
Test avg mae: 0.036186

ZMST
test (2785, 17) 100
using random window
Test avg loss: 0.025525
Test avg mae: 0.040159

test (2785, 17) 100
using random window
Test avg loss: 0.025413
Test avg mae: 0.036667

test (2785, 17) 100
using random window
Test avg loss: 0.025533
Test avg mae: 0.038624

test (2785, 17) 100
using random window
Test avg loss: 0.025489
Test avg mae: 0.038254

test (2785, 17) 100
using random window
Test avg loss: 0.025375
Test avg mae: 0.037249

test (2785, 17) 100
using random window
Test avg loss: 0.025236
Test avg mae: 0.037513

test (2785, 17) 100
using random window
Test avg loss: 0.025373
Test avg mae: 0.037354

test (2785, 17) 100
using random window
Test avg loss: 0.025520
Test avg mae: 0.051640

test (2785, 17) 100
using random window
Test avg loss: 0.025388
Test avg mae: 0.037566

test (2785, 17) 100
using random window
Test avg loss: 0.025274
Test avg mae: 0.038201

LUBW
test (2785, 17) 100
using random window
Test avg loss: 0.025608
Test avg mae: 0.063000

test (2785, 17) 100
using random window
Test avg loss: 0.025088
Test avg mae: 0.042000

test (2785, 17) 100
using random window
Test avg loss: 0.025370
Test avg mae: 0.044000

test (2785, 17) 100
using random window
Test avg loss: 0.025287
Test avg mae: 0.055000

test (2785, 17) 100
using random window
Test avg loss: 0.024473
Test avg mae: 0.033000

test (2785, 17) 100
using random window
Test avg loss: 0.024490
Test avg mae: 0.039000

test (2785, 17) 100
using random window
Test avg loss: 0.026141
Test avg mae: 0.061000

test (2785, 17) 100
using random window
Test avg loss: 0.025247
Test avg mae: 0.048000

test (2785, 17) 100
using random window
Test avg loss: 0.025590
Test avg mae: 0.050000

test (2785, 17) 100
using random window
Test avg loss: 0.025214
Test avg mae: 0.046000

DWOL
test (2785, 17) 100
using random window
Test avg loss: 0.024828
Test avg mae: 0.036150

test (2785, 17) 100
using random window
Test avg loss: 0.024786
Test avg mae: 0.037000

test (2785, 17) 100
using random window
Test avg loss: 0.024687
Test avg mae: 0.035150

test (2785, 17) 100
using random window
Test avg loss: 0.024730
Test avg mae: 0.036050

test (2785, 17) 100
using random window
Test avg loss: 0.024863
Test avg mae: 0.036000

test (2785, 17) 100
using random window
Test avg loss: 0.024617
Test avg mae: 0.037150

test (2785, 17) 100
using random window
Test avg loss: 0.024813
Test avg mae: 0.038150

test (2785, 17) 100
using random window
Test avg loss: 0.024747
Test avg mae: 0.035500

test (2785, 17) 100
using random window
Test avg loss: 0.024657
Test avg mae: 0.036600

test (2785, 17) 100
using random window
Test avg loss: 0.024653
Test avg mae: 0.036100

LUBZ
test (2785, 17) 100
using random window
Test avg loss: 0.023514
Test avg mae: 0.020000

test (2785, 17) 100
using random window
Test avg loss: 0.023465
Test avg mae: 0.010000

test (2785, 17) 100
using random window
Test avg loss: 0.023750
Test avg mae: 0.030000

test (2785, 17) 100
using random window
Test avg loss: 0.023617
Test avg mae: 0.000000

test (2785, 17) 100
using random window
Test avg loss: 0.023518
Test avg mae: 0.000000

test (2785, 17) 100
using random window
Test avg loss: 0.023535
Test avg mae: 0.010000

test (2785, 17) 100
using random window
Test avg loss: 0.023519
Test avg mae: 0.000000

test (2785, 17) 100
using random window
Test avg loss: 0.023594
Test avg mae: 0.010000

test (2785, 17) 100
using random window
Test avg loss: 0.023488
Test avg mae: 0.010000

test (2785, 17) 100
using random window
Test avg loss: 0.023619
Test avg mae: 0.010000

ZUKW2
test (2785, 17) 100
using random window
Test avg loss: 0.024833
Test avg mae: 0.034316

test (2785, 17) 100
using random window
Test avg loss: 0.024815
Test avg mae: 0.034789

test (2785, 17) 100
using random window
Test avg loss: 0.024881
Test avg mae: 0.034526

test (2785, 17) 100
using random window
Test avg loss: 0.024811
Test avg mae: 0.035632

test (2785, 17) 100
using random window
Test avg loss: 0.024757
Test avg mae: 0.032316

test (2785, 17) 100
using random window
Test avg loss: 0.024949
Test avg mae: 0.034000

test (2785, 17) 100
using random window
Test avg loss: 0.024804
Test avg mae: 0.034474

test (2785, 17) 100
using random window
Test avg loss: 0.024805
Test avg mae: 0.034789

test (2785, 17) 100
using random window
Test avg loss: 0.024807
Test avg mae: 0.036105

test (2785, 17) 100
using random window
Test avg loss: 0.024812
Test avg mae: 0.035316

DABR
test (2785, 17) 100
using random window
Test avg loss: 0.024772
Test avg mae: 0.039000

test (2785, 17) 100
using random window
Test avg loss: 0.025039
Test avg mae: 0.040500

test (2785, 17) 100
using random window
Test avg loss: 0.025131
Test avg mae: 0.037250

test (2785, 17) 100
using random window
Test avg loss: 0.025016
Test avg mae: 0.038500

test (2785, 17) 100
using random window
Test avg loss: 0.025114
Test avg mae: 0.039125

test (2785, 17) 100
using random window
Test avg loss: 0.024994
Test avg mae: 0.038688

test (2785, 17) 100
using random window
Test avg loss: 0.025277
Test avg mae: 0.038375

test (2785, 17) 100
using random window
Test avg loss: 0.024936
Test avg mae: 0.038125

test (2785, 17) 100
using random window
Test avg loss: 0.024811
Test avg mae: 0.038500

test (2785, 17) 100
using random window
Test avg loss: 0.025205
Test avg mae: 0.039000

PEKW2
test (2785, 17) 100
using random window
Test avg loss: 0.024234
Test avg mae: 0.036897

test (2785, 17) 100
using random window
Test avg loss: 0.024220
Test avg mae: 0.037816

test (2785, 17) 100
using random window
Test avg loss: 0.024231
Test avg mae: 0.037931

test (2785, 17) 100
using random window
Test avg loss: 0.024209
Test avg mae: 0.036207

test (2785, 17) 100
using random window
Test avg loss: 0.024214
Test avg mae: 0.037241

test (2785, 17) 100
using random window
Test avg loss: 0.024192
Test avg mae: 0.038966

test (2785, 17) 100
using random window
Test avg loss: 0.024206
Test avg mae: 0.036207

test (2785, 17) 100
using random window
Test avg loss: 0.024220
Test avg mae: 0.034828

test (2785, 17) 100
using random window
Test avg loss: 0.024178
Test avg mae: 0.037126

test (2785, 17) 100
using random window
Test avg loss: 0.024201
Test avg mae: 0.040460

KRZY
test (2785, 17) 100
using random window
Test avg loss: 0.025639
Test avg mae: 0.070000

test (2785, 17) 100
using random window
Test avg loss: 0.024752
Test avg mae: 0.050000

test (2785, 17) 100
using random window
Test avg loss: 0.024743
Test avg mae: 0.060000

test (2785, 17) 100
using random window
Test avg loss: 0.025550
Test avg mae: 0.070000

test (2785, 17) 100
using random window
Test avg loss: 0.025862
Test avg mae: 0.060000

test (2785, 17) 100
using random window
Test avg loss: 0.024733
Test avg mae: 0.050000

test (2785, 17) 100
using random window
Test avg loss: 0.025121
Test avg mae: 0.060000

test (2785, 17) 100
using random window
Test avg loss: 0.026550
Test avg mae: 0.080000

test (2785, 17) 100
using random window
Test avg loss: 0.024653
Test avg mae: 0.050000

test (2785, 17) 100
using random window
Test avg loss: 0.025185
Test avg mae: 0.070000

OBIS
test (2785, 17) 100
using random window
Test avg loss: 0.025883
Test avg mae: 0.054245

test (2785, 17) 100
using random window
Test avg loss: 0.026269
Test avg mae: 0.054717

test (2785, 17) 100
using random window
Test avg loss: 0.025488
Test avg mae: 0.054245

test (2785, 17) 100
using random window
Test avg loss: 0.025862
Test avg mae: 0.042830

test (2785, 17) 100
using random window
Test avg loss: 0.025587
Test avg mae: 0.056509

test (2785, 17) 100
using random window
Test avg loss: 0.026170
Test avg mae: 0.055566

test (2785, 17) 100
using random window
Test avg loss: 0.025503
Test avg mae: 0.057925

test (2785, 17) 100
using random window
Test avg loss: 0.025792
Test avg mae: 0.055755

test (2785, 17) 100
using random window
Test avg loss: 0.025840
Test avg mae: 0.052736

test (2785, 17) 100
using random window
Test avg loss: 0.025825
Test avg mae: 0.056415

KAZI
test (2785, 17) 100
using random window
Test avg loss: 0.025883
Test avg mae: 0.044649

test (2785, 17) 100
using random window
Test avg loss: 0.025872
Test avg mae: 0.042982

test (2785, 17) 100
using random window
Test avg loss: 0.025842
Test avg mae: 0.042281

test (2785, 17) 100
using random window
Test avg loss: 0.025850
Test avg mae: 0.040789

test (2785, 17) 100
using random window
Test avg loss: 0.025822
Test avg mae: 0.041930

test (2785, 17) 100
using random window
Test avg loss: 0.025798
Test avg mae: 0.041491

test (2785, 17) 100
using random window
Test avg loss: 0.025872
Test avg mae: 0.041404

test (2785, 17) 100
using random window
Test avg loss: 0.025900
Test avg mae: 0.041316

test (2785, 17) 100
using random window
Test avg loss: 0.025843
Test avg mae: 0.042632

test (2785, 17) 100
using random window
Test avg loss: 0.025867
Test avg mae: 0.042368

KWLC
test (2785, 17) 100
using random window
Test avg loss: 0.025280
Test avg mae: 0.051923

test (2785, 17) 100
using random window
Test avg loss: 0.025302
Test avg mae: 0.050000

test (2785, 17) 100
using random window
Test avg loss: 0.025410
Test avg mae: 0.053462

test (2785, 17) 100
using random window
Test avg loss: 0.025195
Test avg mae: 0.048269

test (2785, 17) 100
using random window
Test avg loss: 0.025163
Test avg mae: 0.049423

test (2785, 17) 100
using random window
Test avg loss: 0.025168
Test avg mae: 0.048269

test (2785, 17) 100
using random window
Test avg loss: 0.025206
Test avg mae: 0.050385

test (2785, 17) 100
using random window
Test avg loss: 0.025152
Test avg mae: 0.048846

test (2785, 17) 100
using random window
Test avg loss: 0.025373
Test avg mae: 0.054615

test (2785, 17) 100
using random window
Test avg loss: 0.025162
Test avg mae: 0.050192

Plot results

In [8]:

results_df = []
for i, split in enumerate(splits): 
    df = pd.DataFrame(results[i]).transpose()

    for station, values in df[['mae']].itertuples():
        df.loc[station, 'mae'] =  np.mean([v for v in values if v is not None]) 
    for station, values in df[['loss']].itertuples():
        df.loc[station, 'loss'] =  np.mean([v for v in values if v is not None]) 
        
    df.plot(kind='bar', figsize=(15,4), title=f"Mean results per station in {split} set")

    results_df.append(df)

/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3464: RuntimeWarning: Mean of empty slice.
  return _methods._mean(a, axis=axis, dtype=dtype,
/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide
  ret = ret.dtype.type(ret / rcount)
/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3464: RuntimeWarning: Mean of empty slice.
  return _methods._mean(a, axis=axis, dtype=dtype,
/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide
  ret = ret.dtype.type(ret / rcount)
/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3464: RuntimeWarning: Mean of empty slice.
  return _methods._mean(a, axis=axis, dtype=dtype,
/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide
  ret = ret.dtype.type(ret / rcount)
/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3464: RuntimeWarning: Mean of empty slice.
  return _methods._mean(a, axis=axis, dtype=dtype,
/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide
  ret = ret.dtype.type(ret / rcount)
/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3464: RuntimeWarning: Mean of empty slice.
  return _methods._mean(a, axis=axis, dtype=dtype,
/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide
  ret = ret.dtype.type(ret / rcount)
/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3464: RuntimeWarning: Mean of empty slice.
  return _methods._mean(a, axis=axis, dtype=dtype,
/Users/krystynamilian/virtualenvs/epos/lib/python3.9/site-packages/numpy/core/_methods.py:192: RuntimeWarning: invalid value encountered in scalar divide
  ret = ret.dtype.type(ret / rcount)

In [9]:

### Check correlation between trainin data size and obtained results

In [10]:

stats = frames_per_station.copy()

In [11]:

stats['train_mae'] = results_df[0]['mae']
stats['dev_mae'] = results_df[1]['mae']
stats['test_mae'] = results_df[2]['mae']

In [12]:

stats.plot(kind='scatter', x ='train', y='train_mae')

Out [12]:

<Axes: xlabel='train', ylabel='train_mae'>

In [13]:

stats

Out [13]:

	train	dev	test	train_mae	dev_mae	test_mae
station_code
BRDW	160.0	20.0	67.0	0.092019	0.08675	0.065433
DABR	359.0	161.0	160.0	0.034323	0.036516	0.038706
DWOL	479.0	206.0	200.0	0.02986	0.039102	0.036385
GROD	1052.0	197.0	175.0	0.05752	0.049132	0.047029
GUZI	740.0	121.0	126.0	0.035936	0.038777	0.050024
JEDR	809.0	9.0	62.0	0.035391	0.013	0.073871
KAZI	105.0	110.0	114.0	0.026905	0.029736	0.042184
KRZY	7.0	11.0	1.0	0.056	0.101909	0.062
LUBW	33.0	24.0	10.0	0.090697	0.115	0.0481
LUBZ	2.0	3.0	1.0	0.1685	0.015	0.01
MOSK2	958.0	197.0	199.0	0.040543	0.04536	0.035337
NWLU	902.0	138.0	149.0	0.03412	0.036971	0.030188
OBIS	145.0	97.0	106.0	0.02649	0.053753	0.054094
PCHB	420.0	82.0	79.0	0.040777	0.042024	0.039608
PEKW2	205.0	97.0	87.0	0.043195	0.04801	0.037368
PPOL	463.0	67.0	59.0	0.064004	0.06909	0.048424
RUDN	941.0	172.0	152.0	0.047671	0.049262	0.055829
RYNR	874.0	151.0	135.0	0.065755	0.068841	0.087104
RZEC	29.0	NaN	NaN	0.028448	NaN	NaN
SGOR	845.0	155.0	158.0	0.036054	0.068084	0.043418
TRBC2	295.0	59.0	55.0	0.044679	0.032559	0.033709
TRN2	1020.0	181.0	162.0	0.053552	0.040359	0.067352
TRZS	209.0	116.0	97.0	0.044144	0.115612	0.036155
ZMST	1084.0	191.0	189.0	0.049537	0.030513	0.039323
ZUKW2	308.0	206.0	190.0	0.033697	0.038471	0.034626
KWLC	NaN	2.0	52.0	NaN	0.021	0.050538

Check predictions for stations with highest MAE

In [14]:

dev_res = results_df[1]
station_with_worst_res_dev_set = dev_res[dev_res.mae == dev_res.mae.max()].index[0]
highest_dev_mae = dev_res.loc[station_with_worst_res_dev_set, 'mae']

test_res = results_df[2]
station_with_worst_res_test_set = test_res[test_res.mae == test_res.mae.max()].index[0]
highest_test_mae = test_res.loc[station_with_worst_res_test_set, 'mae']
print(f"highest mean MAE in dev set: {highest_dev_mae:.2f} for station: {station_with_worst_res_dev_set}")
print(f"highest mean MAE in test set: {highest_test_mae:.2f} for station: {station_with_worst_res_test_set}")

highest mean MAE in dev set: 0.12 for station: TRZS
highest mean MAE in test set: 0.09 for station: RYNR

In [15]:

def plot_sample(sample, model, i, desc=None): 
    fig = plt.figure(figsize=(15, 10))
    
    axs = fig.subplots(2, 1, sharex=True, gridspec_kw={"hspace": 0, "height_ratios": [3,  2]})
    axs[0].plot(sample["X"][0].T, label='x')
    plt.legend()
    axs[1].plot(sample["y"][0].T, label='y')
    
    model.eval()  # close the model for evaluation
    
    with torch.no_grad():
        pred = model(torch.tensor(sample["X"], device=model.device).unsqueeze(0))  # Add a fake batch dimension
        pred = pred[0].cpu().numpy()
        
        axs[1].plot(pred[0], label='pred', color='orange')
        plt.legend()

        pred_pick_idx  = np.argmax(pred[0])
        true_pick_idx = np.argmax(sample['y'][0])

        
        
        mae_error = np.abs(pred_pick_idx - true_pick_idx) /100 #mae in seconds

        fig.suptitle(f"Predictions for sample: {i} {desc}, mae: {mae_error}s")
        
        plt.show()
        
    

Find random samples that reproduce obtained results

Results are not deterministic, because samples generator used during training augments samples by introducing random padding, see https://seisbench.readthedocs.io/en/stable/pages/documentation/generate.html?highlight=generate#seisbench.generate.windows.RandomWindow

In [16]:

##### dev set

In [17]:

mean_mae = 0
samples = []
split = 'dev'
station = station_with_worst_res_dev_set

while mean_mae < highest_dev_mae: 

    gen = train.get_data_generator(split=split, station=station , sampling_rate=sampling_rate, path=data_path)
    station_mae = []
    with torch.no_grad():
        for i in range(len(gen)): 
            # idx = np.random.randint(len(gen))
            idx = i
            sample = gen[idx]
            samples.append(sample)
            pred = model(torch.tensor(sample["X"], device=model.device).unsqueeze(0))  
            pred = pred[0].cpu().numpy()
    
            pred_pick_idx  = np.argmax(pred[0])
            true_pick_idx = np.argmax(sample['y'][0])            
                
            mae_error = np.abs(pred_pick_idx - true_pick_idx) /100 #mae in seconds
            station_mae.append(mae_error)
    
    sorted = np.argsort(station_mae)[::-1]
    mean_mae = np.mean(station_mae)

print(np.array(station_mae)[sorted])


## plot samples with mae error at leas 0.2s
for idx in sorted:
    if station_mae[idx] < 0.2: 
        break
    print(idx, station_mae[idx])
    plot_sample(samples[idx], model, idx, desc=f" from station {station} {split} set")

dev (2773, 17) 100
using random window
dev (2773, 17) 100
using random window
dev (2773, 17) 100
using random window
dev (2773, 17) 100
using random window
dev (2773, 17) 100
using random window
dev (2773, 17) 100
using random window
dev (2773, 17) 100
using random window
dev (2773, 17) 100
using random window

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

[1.594e+01 4.900e-01 4.500e-01 2.600e-01 2.000e-01 1.300e-01 1.200e-01
 1.200e-01 1.000e-01 9.000e-02 9.000e-02 8.000e-02 8.000e-02 6.000e-02
 6.000e-02 6.000e-02 6.000e-02 6.000e-02 6.000e-02 6.000e-02 5.000e-02
 5.000e-02 5.000e-02 5.000e-02 4.000e-02 4.000e-02 4.000e-02 4.000e-02
 4.000e-02 4.000e-02 4.000e-02 4.000e-02 4.000e-02 4.000e-02 3.000e-02
 3.000e-02 3.000e-02 3.000e-02 3.000e-02 3.000e-02 3.000e-02 3.000e-02
 3.000e-02 3.000e-02 3.000e-02 3.000e-02 3.000e-02 3.000e-02 3.000e-02
 3.000e-02 3.000e-02 3.000e-02 3.000e-02 2.000e-02 2.000e-02 2.000e-02
 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02
 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02
 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02 2.000e-02 1.000e-02
 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02
 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02
 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02 1.000e-02
 1.000e-02 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00
 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00
 0.000e+00 0.000e+00 0.000e+00 0.000e+00]
59 15.94

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

0 0.49

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

53 0.45

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

110 0.26

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

64 0.2

In [18]:

mean_mae = 0
samples = []
split = 'test'
station = station_with_worst_res_test_set

while mean_mae < highest_test_mae: 

    gen = train.get_data_generator(split=split, station=station , sampling_rate=sampling_rate, path=data_path)
    station_mae = []
    with torch.no_grad():
        for i in range(len(gen)): 
            # idx = np.random.randint(len(gen))
            idx = i
            sample = gen[idx]
            samples.append(sample)
            pred = model(torch.tensor(sample["X"], device=model.device).unsqueeze(0))  
            pred = pred[0].cpu().numpy()
    
            pred_pick_idx  = np.argmax(pred[0])
            true_pick_idx = np.argmax(sample['y'][0])            
                
            mae_error = np.abs(pred_pick_idx - true_pick_idx) /100 #mae in seconds
            station_mae.append(mae_error)
    
    sorted = np.argsort(station_mae)[::-1]
    mean_mae = np.mean(station_mae)

print(sorted)
print(np.array(station_mae)[sorted])


## plot samples with mae error at leas 0.2s
for idx in sorted:
    if station_mae[idx] < 0.2: 
        break
    print(idx, station_mae[idx])
    plot_sample(samples[idx], model, idx, desc=f" from station {station} {split} set")

test (2785, 17) 100
using random window
test (2785, 17) 100
using random window

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

[ 91  40  63 106  28 119 100  33 129 105  42  54  86  32 130  24  61   3
  79  73  67  89 128 124  76  50  46  36  71  31   5  83  98 126  18  21
  77   2 134 114 127 109 125  22  30  23  20  81  15  43  44  45 113  49
  96  51  74  94 117  52 118  95 122  57 123  14  13  12   9   8   7   6
 131 132  56  47  65  93  90  97 110  87  75  85 108 102  80  59  25 104
  19  58  17  82  84  11  10   4 120   0  68  55  48  69 112  53  38  39
  37  60 116  66 111  92 101  88  62  99  64   1  34 133  16  35  70  41
  78 103 115  26  27 121 107  72  29]
[3.12 2.56 1.23 0.5  0.37 0.23 0.22 0.14 0.13 0.12 0.11 0.1  0.1  0.09
 0.09 0.08 0.08 0.08 0.07 0.07 0.07 0.07 0.07 0.06 0.06 0.06 0.06 0.06
 0.05 0.05 0.05 0.05 0.05 0.05 0.04 0.04 0.04 0.04 0.04 0.04 0.04 0.04
 0.04 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03 0.03
 0.03 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02
 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02 0.02
 0.02 0.02 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01
 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01
 0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.   0.
 0.   0.   0.   0.   0.   0.   0.   0.   0.  ]
91 3.12

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

40 2.56

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

63 1.23

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

106 0.5

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

28 0.37

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

119 0.23

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

100 0.22

In [19]:

data.test().metadata.iloc[idx]

Out [19]:

index                                            30312
source_origin_time             2021-08-24 03:01:23.500
source_latitude_deg                      5714247.36402
source_longitude_deg                    5578128.427708
source_depth_km                                    0.8
source_magnitude                              1.522222
split                                             test
station_network_code                                PL
station_code                                     MOSK2
trace_channel                                      EHE
trace_sampling_rate_hz                           100.0
trace_start_time           2021-08-24T03:01:17.360000Z
trace_Pg_arrival_sample                          727.0
trace_name                        bucket29$66,:3,:2001
trace_Sg_arrival_sample                            NaN
trace_chunk                                           
trace_component_order                              ZNE
Name: 30312, dtype: object

In [20]:

idx

Out [20]:

33

In [21]:

gen = train.get_data_generator(split='test', station=None , sampling_rate=sampling_rate, path=data_path)

for i in range(5): 
    idx = np.random.randint(len(gen))
    sample = gen[idx]
    
    plot_sample(sample, model, idx, desc="")

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

test (2785, 17) 100
using random window

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.

In [22]:

sample

Out [22]:

{'X': array([[-2.9614418e-19, -2.9614418e-19, -2.9614418e-19, ...,
         -2.9614418e-19, -2.9614418e-19, -2.9614418e-19],
        [-6.4017395e-19, -6.4017395e-19, -6.4017395e-19, ...,
         -6.4017395e-19, -6.4017395e-19, -6.4017395e-19],
        [-1.0255816e-18, -1.0255816e-18, -1.0255816e-18, ...,
         -1.0255816e-18, -1.0255816e-18, -1.0255816e-18]], dtype=float32),
 'y': array([[2.43596292e-226, 7.12942807e-226, 2.08428048e-225, ...,
         0.00000000e+000, 0.00000000e+000, 0.00000000e+000],
        [1.00000000e+000, 1.00000000e+000, 1.00000000e+000, ...,
         1.00000000e+000, 1.00000000e+000, 1.00000000e+000]])}

In [ ]: