{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c65c7056-5b31-42f8-8278-9c9c2b636187",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from obspy import read\n",
    "from  matplotlib import pyplot as plt, dates \n",
    "\n",
    "import glob"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea411d66-aaf1-48e0-9d19-ed5560432002",
   "metadata": {},
   "source": [
    "# Explore data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ac416bcb-6e94-49dd-be68-cedef82dcefc",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path=\"../data/igf/\"\n",
    "data = pd.read_excel(data_path + \"Catalog_20_21.xlsx\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "41064808-0bd0-45ee-9c1e-21e465c823cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Datetime</th>\n",
       "      <th>X</th>\n",
       "      <th>Y</th>\n",
       "      <th>Depth</th>\n",
       "      <th>Mw</th>\n",
       "      <th>Phases</th>\n",
       "      <th>mseed_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-01-01 10:09:42.200</td>\n",
       "      <td>5.582503e+06</td>\n",
       "      <td>5.702646e+06</td>\n",
       "      <td>0.7</td>\n",
       "      <td>2.469231</td>\n",
       "      <td>Pg BRDW 2020-01-01 10:09:44.400, Sg BRDW 2020-...</td>\n",
       "      <td>20200101100941.mseed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-01-02 08:03:06.300</td>\n",
       "      <td>5.577930e+06</td>\n",
       "      <td>5.705644e+06</td>\n",
       "      <td>0.7</td>\n",
       "      <td>1.916667</td>\n",
       "      <td>Pg BRDW 2020-01-02 08:03:07.118, Sg BRDW 2020-...</td>\n",
       "      <td>20200102080305.mseed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-01-03 14:36:51.500</td>\n",
       "      <td>5.582597e+06</td>\n",
       "      <td>5.701562e+06</td>\n",
       "      <td>0.7</td>\n",
       "      <td>1.916667</td>\n",
       "      <td>Pg DWOL 2020-01-03 14:36:54.525, Sg DWOL 2020-...</td>\n",
       "      <td>20200103143651.mseed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-01-04 05:01:01.700</td>\n",
       "      <td>5.575319e+06</td>\n",
       "      <td>5.716387e+06</td>\n",
       "      <td>0.7</td>\n",
       "      <td>1.600000</td>\n",
       "      <td>Pg DWOL 2020-01-04 05:01:03.089, Sg DWOL 2020-...</td>\n",
       "      <td>20200104050102.mseed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-01-04 14:52:08.100</td>\n",
       "      <td>5.577217e+06</td>\n",
       "      <td>5.705684e+06</td>\n",
       "      <td>0.7</td>\n",
       "      <td>1.844444</td>\n",
       "      <td>Pg BRDW 2020-01-04 14:52:08.488, Sg BRDW 2020-...</td>\n",
       "      <td>20200104145207.mseed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Datetime             X             Y  Depth        Mw   \n",
       "0 2020-01-01 10:09:42.200  5.582503e+06  5.702646e+06    0.7  2.469231  \\\n",
       "1 2020-01-02 08:03:06.300  5.577930e+06  5.705644e+06    0.7  1.916667   \n",
       "2 2020-01-03 14:36:51.500  5.582597e+06  5.701562e+06    0.7  1.916667   \n",
       "3 2020-01-04 05:01:01.700  5.575319e+06  5.716387e+06    0.7  1.600000   \n",
       "4 2020-01-04 14:52:08.100  5.577217e+06  5.705684e+06    0.7  1.844444   \n",
       "\n",
       "                                              Phases            mseed_name  \n",
       "0  Pg BRDW 2020-01-01 10:09:44.400, Sg BRDW 2020-...  20200101100941.mseed  \n",
       "1  Pg BRDW 2020-01-02 08:03:07.118, Sg BRDW 2020-...  20200102080305.mseed  \n",
       "2  Pg DWOL 2020-01-03 14:36:54.525, Sg DWOL 2020-...  20200103143651.mseed  \n",
       "3  Pg DWOL 2020-01-04 05:01:03.089, Sg DWOL 2020-...  20200104050102.mseed  \n",
       "4  Pg BRDW 2020-01-04 14:52:08.488, Sg BRDW 2020-...  20200104145207.mseed  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "668150d2-1a3c-4bbf-9b56-1e47ede6dfd4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2198, 7)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f8cc5794-216b-4105-aa96-42c55af43306",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<DatetimeArray>\n",
       "['2020-01-01 10:09:42.200000', '2020-01-02 08:03:06.300000',\n",
       " '2020-01-03 14:36:51.500000', '2020-01-04 05:01:01.700000',\n",
       " '2020-01-04 14:52:08.100000',        '2020-01-04 19:47:26',\n",
       " '2020-01-05 06:27:21.300000',        '2020-01-05 10:42:54',\n",
       " '2020-01-05 20:56:35.700000', '2020-01-06 05:35:10.100000',\n",
       " ...\n",
       " '2021-10-29 22:13:28.200000', '2021-10-30 09:01:58.100000',\n",
       " '2021-10-30 10:55:19.100000', '2021-10-30 20:49:21.200000',\n",
       " '2021-10-30 21:01:09.800000', '2021-10-30 22:12:27.100000',\n",
       " '2021-10-31 00:03:17.800000', '2021-10-31 01:13:04.100000',\n",
       " '2021-10-31 18:12:38.700000', '2021-10-31 19:11:48.200000']\n",
       "Length: 2198, dtype: datetime64[ns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.Datetime.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "36d27c72-df99-4b23-962c-6eab122eeef1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1720, 7)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna().shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fae3e69a-a5c5-42c9-9575-da54244e7994",
   "metadata": {},
   "source": [
    "## Check missing data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f7d97da-74c8-40d7-bbae-0776379d39b3",
   "metadata": {},
   "source": [
    "#### Missing mseed_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "01e7c1bd-506b-4b3a-9378-b9ce7f91f54c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Datetime</th>\n",
       "      <th>X</th>\n",
       "      <th>Y</th>\n",
       "      <th>Depth</th>\n",
       "      <th>Mw</th>\n",
       "      <th>Phases</th>\n",
       "      <th>mseed_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2020-01-06 18:46:05.900</td>\n",
       "      <td>5.583057e+06</td>\n",
       "      <td>5.701401e+06</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>1.880000</td>\n",
       "      <td>Pg GROD 2020-01-06 18:46:08.904, Sg GROD 2020-...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2020-01-07 19:24:16.900</td>\n",
       "      <td>5.583143e+06</td>\n",
       "      <td>5.701039e+06</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>1.820000</td>\n",
       "      <td>Pg JEDR 2020-01-07 19:24:19.289, Sg JEDR 2020-...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>2020-01-14 03:58:31.700</td>\n",
       "      <td>5.582596e+06</td>\n",
       "      <td>5.701750e+06</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>1.850000</td>\n",
       "      <td>Pg JEDR 2020-01-14 03:58:33.990, Sg JEDR 2020-...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>2020-01-15 09:01:16.700</td>\n",
       "      <td>5.582595e+06</td>\n",
       "      <td>5.703723e+06</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>2.071429</td>\n",
       "      <td>Pg GROD 2020-01-15 09:01:19.262, Sg GROD 2020-...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>2020-01-16 04:55:53.600</td>\n",
       "      <td>5.576011e+06</td>\n",
       "      <td>5.706832e+06</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>1.640000</td>\n",
       "      <td>Pg MOSK2 2020-01-16 04:55:55.566, Sg MOSK2 202...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2166</th>\n",
       "      <td>2021-10-25 00:30:29.400</td>\n",
       "      <td>5.574834e+06</td>\n",
       "      <td>5.706510e+06</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>1.600000</td>\n",
       "      <td>Pg BRDW 2021-10-25 00:30:30.324, Sg BRDW 2021-...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2173</th>\n",
       "      <td>2021-10-26 08:05:39.300</td>\n",
       "      <td>5.575470e+06</td>\n",
       "      <td>5.715877e+06</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>1.366667</td>\n",
       "      <td>Sg DWOL 2021-10-26 08:05:40.992, Pg DWOL 2021-...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2184</th>\n",
       "      <td>2021-10-28 15:15:41.600</td>\n",
       "      <td>5.575855e+06</td>\n",
       "      <td>5.715674e+06</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>1.300000</td>\n",
       "      <td>Pg DWOL 2021-10-28 15:15:42.729, Sg DWOL 2021-...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2186</th>\n",
       "      <td>2021-10-28 23:28:26.400</td>\n",
       "      <td>5.577772e+06</td>\n",
       "      <td>5.707358e+06</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1.400000</td>\n",
       "      <td>Pg DABR 2021-10-28 23:28:28.434, Sg DABR 2021-...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2193</th>\n",
       "      <td>2021-10-30 22:12:27.100</td>\n",
       "      <td>5.580585e+06</td>\n",
       "      <td>5.713451e+06</td>\n",
       "      <td>0.919473</td>\n",
       "      <td>1.483333</td>\n",
       "      <td>Pg ZUKW2 2021-10-30 22:12:27.983, Sg ZUKW2 202...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>478 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Datetime             X             Y     Depth        Mw   \n",
       "11   2020-01-06 18:46:05.900  5.583057e+06  5.701401e+06  0.700000  1.880000  \\\n",
       "14   2020-01-07 19:24:16.900  5.583143e+06  5.701039e+06  0.700000  1.820000   \n",
       "31   2020-01-14 03:58:31.700  5.582596e+06  5.701750e+06  0.700000  1.850000   \n",
       "36   2020-01-15 09:01:16.700  5.582595e+06  5.703723e+06  0.800000  2.071429   \n",
       "38   2020-01-16 04:55:53.600  5.576011e+06  5.706832e+06  0.700000  1.640000   \n",
       "...                      ...           ...           ...       ...       ...   \n",
       "2166 2021-10-25 00:30:29.400  5.574834e+06  5.706510e+06  0.700000  1.600000   \n",
       "2173 2021-10-26 08:05:39.300  5.575470e+06  5.715877e+06  0.900000  1.366667   \n",
       "2184 2021-10-28 15:15:41.600  5.575855e+06  5.715674e+06  0.900000  1.300000   \n",
       "2186 2021-10-28 23:28:26.400  5.577772e+06  5.707358e+06  0.500000  1.400000   \n",
       "2193 2021-10-30 22:12:27.100  5.580585e+06  5.713451e+06  0.919473  1.483333   \n",
       "\n",
       "                                                 Phases mseed_name  \n",
       "11    Pg GROD 2020-01-06 18:46:08.904, Sg GROD 2020-...        NaN  \n",
       "14    Pg JEDR 2020-01-07 19:24:19.289, Sg JEDR 2020-...        NaN  \n",
       "31    Pg JEDR 2020-01-14 03:58:33.990, Sg JEDR 2020-...        NaN  \n",
       "36    Pg GROD 2020-01-15 09:01:19.262, Sg GROD 2020-...        NaN  \n",
       "38    Pg MOSK2 2020-01-16 04:55:55.566, Sg MOSK2 202...        NaN  \n",
       "...                                                 ...        ...  \n",
       "2166  Pg BRDW 2021-10-25 00:30:30.324, Sg BRDW 2021-...        NaN  \n",
       "2173  Sg DWOL 2021-10-26 08:05:40.992, Pg DWOL 2021-...        NaN  \n",
       "2184  Pg DWOL 2021-10-28 15:15:42.729, Sg DWOL 2021-...        NaN  \n",
       "2186  Pg DABR 2021-10-28 23:28:28.434, Sg DABR 2021-...        NaN  \n",
       "2193  Pg ZUKW2 2021-10-30 22:12:27.983, Sg ZUKW2 202...        NaN  \n",
       "\n",
       "[478 rows x 7 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[(data.mseed_name.isna())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "28b1a93f-8342-4eba-86b2-a3724a5c038c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing mseed fname in catalog:  478\n"
     ]
    }
   ],
   "source": [
    "print(\"Missing mseed fname in catalog: \", data[(data.mseed_name.isna())].shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9bbe4ea7-c86c-4874-8b6d-93578d30e991",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_data_from_catalog(fname): \n",
    "    targets, frames = [], []\n",
    "    data = data = pd.read_excel(fname, index_col=0)\n",
    "    for row in data: \n",
    "        _targets, _frames = extract_data_from_event(row.Phases, row.mseed_fname)\n",
    "    targets.extend(_targets)\n",
    "    frames.extend(_frames)\n",
    "    return targets, frames\n",
    "\n",
    "\n",
    "def extract_data_from_event(phases, mseed_fname): \n",
    "    pass"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1f9caf3-caa0-4a87-a450-8025eee85e77",
   "metadata": {},
   "source": [
    "### Calculate stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "95691a54-46be-4d46-a779-3d5dfebc0d2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_stats(phases, mseed_fname, stats={}, path=data_path+\"mseeds\"): \n",
    "    \n",
    "    phases = [ann.split(' ') for ann in phases.split(', ')]\n",
    "    phases = pd.DataFrame(phases, columns = ['phase', 'station', 'date', 'hour'])\n",
    "    phases.index = pd.DatetimeIndex(phases.date + ' ' + phases.hour, tz= \"UTC\")\n",
    "\n",
    "    \n",
    "    mseed_path = f\"{path}/mseeds_2020/{mseed_fname}\"\n",
    "    if not os.path.exists(mseed_path): \n",
    "        mseed_path = f\"{path}/mseeds_2021/{mseed_fname}\"\n",
    "\n",
    "    mseed_exisits = os.path.exists(mseed_path)\n",
    "\n",
    "    missing_traces_count = 0 # phases.shape[0]/2\n",
    "    missing_annotations_count = 0\n",
    "    missing_traces = {}\n",
    "\n",
    "    if mseed_exisits: \n",
    "\n",
    "        stream = read(mseed_path)\n",
    "        traces_count = len(stream.traces)\n",
    "        \n",
    "        mseed_stations = set()\n",
    "        \n",
    "        for i, trace in enumerate(stream.traces): \n",
    "           \n",
    "            station = trace.stats.station\n",
    "            mseed_stations.add(station)\n",
    "            station_annotations = phases[phases.station==station]\n",
    "            p_timestamps = station_annotations[station_annotations.phase=='Pg'].index\n",
    "            s_timestamps = station_annotations[station_annotations.phase=='Sg'].index\n",
    "\n",
    "\n",
    "            if station not in stats: \n",
    "                stats[station] = {'p':0, 's':0}\n",
    "            stats[station]['p'] += len(p_timestamps)\n",
    "            stats[station]['s'] += len(s_timestamps)\n",
    "\n",
    "\n",
    "        annotated_stations = set(phases.station.values)\n",
    "        stations_with_missing_traces = annotated_stations - mseed_stations\n",
    "        missing_traces_count = len(stations_with_missing_traces)\n",
    "        \n",
    "        stations_with_missing_annotations = mseed_stations - annotated_stations\n",
    "        missing_annotations_count = len(stations_with_missing_annotations)\n",
    "        \n",
    "        missing_data = {\n",
    "            mseed_path: {\n",
    "                'stations_with_missing_traces': stations_with_missing_traces, \n",
    "                'stations_with_missing_annotations': stations_with_missing_annotations,\n",
    "                'stations_with_available_data': mseed_stations.intersection(annotated_stations)\n",
    "            }\n",
    "        }\n",
    "\n",
    "    return stats, mseed_exisits, missing_annotations_count, missing_traces_count, missing_data\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7b4a17f1-65a0-4636-96fd-670ac1fb8505",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stats\n",
      "Missing files:  0\n",
      "Missing annotations:  12286\n",
      "Missing traces:  142\n",
      "Missing mseed fname in catalog:  478\n"
     ]
    }
   ],
   "source": [
    "stats = {}\n",
    "missing_files = []\n",
    "missing_traces = 0\n",
    "missing_annotations = 0 \n",
    "missing_data = {}\n",
    "\n",
    "for phases, fname in zip(data.Phases, data.mseed_name): \n",
    "    \n",
    "    if not pd.isna(fname): \n",
    "        stats, mseed_exisits, missing_annotations_, missing_traces_, missing_data_ = calculate_stats(phases, fname, stats)\n",
    "        missing_annotations += missing_annotations_\n",
    "        missing_traces += missing_traces_\n",
    "        missing_data.update(missing_data_)\n",
    "    if not mseed_exisits: \n",
    "        missing_files.append(fname)\n",
    "        \n",
    "stats = pd.DataFrame(stats)/3        \n",
    "missing_data = pd.DataFrame(missing_data).transpose()\n",
    "\n",
    "print(\"Stats\")\n",
    "print(\"Missing files: \", len(missing_files))\n",
    "print(\"Missing annotations: \", missing_annotations)\n",
    "print(\"Missing traces: \", missing_traces)\n",
    "print(\"Missing mseed fname in catalog: \", data[(data.mseed_name.isna())].shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "30139ddc-727d-4494-b33b-ec4068e6061d",
   "metadata": {},
   "outputs": [],
   "source": [
    "missing_data.stations_with_missing_traces = missing_data.stations_with_missing_traces.apply(lambda x: list(x) if len(x)>0 else None)\n",
    "missing_data.stations_with_missing_annotations = missing_data.stations_with_missing_annotations.apply(lambda x: list(x) if len(x)>0 else None)\n",
    "missing_data['path'] = missing_data.index\n",
    "missing_data.index = missing_data.path.apply(lambda x: os.path.basename(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c8ecb508-430c-4c45-800c-285f3b6eef91",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stations_with_missing_traces</th>\n",
       "      <th>stations_with_missing_annotations</th>\n",
       "      <th>stations_with_available_data</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>path</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20200109224901.mseed</th>\n",
       "      <td>[RYNR]</td>\n",
       "      <td>[RZEC, TRBC2, LUBW, TRZS, PPOL, OBIS, GROD, RU...</td>\n",
       "      <td>{MOSK2, JEDR, DWOL, BRDW, GUZI, NWLU, TRN2, PC...</td>\n",
       "      <td>../data/igf/mseeds/mseeds_2020/20200109224901....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20200111165827.mseed</th>\n",
       "      <td>[TRN2]</td>\n",
       "      <td>[RZEC, TRBC2, LUBW, TRZS, OBIS, RUDN, LUBZ]</td>\n",
       "      <td>{MOSK2, JEDR, DWOL, RYNR, BRDW, GUZI, NWLU, PP...</td>\n",
       "      <td>../data/igf/mseeds/mseeds_2020/20200111165827....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20200112055037.mseed</th>\n",
       "      <td>[TRZS]</td>\n",
       "      <td>[RZEC, TRBC2, MOSK2, LUBW, BRDW, SGOR, GUZI, P...</td>\n",
       "      <td>{JEDR, DWOL, RYNR, NWLU, RUDN, TRN2, GROD, ZMST}</td>\n",
       "      <td>../data/igf/mseeds/mseeds_2020/20200112055037....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20200114011249.mseed</th>\n",
       "      <td>[PCHB]</td>\n",
       "      <td>[RZEC, TRBC2, LUBW, DWOL, BRDW, TRZS, SGOR, GU...</td>\n",
       "      <td>{MOSK2, JEDR, RYNR, PPOL, RUDN, TRN2, GROD, ZMST}</td>\n",
       "      <td>../data/igf/mseeds/mseeds_2020/20200114011249....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20200123060736.mseed</th>\n",
       "      <td>[GROD]</td>\n",
       "      <td>[RZEC, TRBC2, LUBW, BRDW, OBIS, PCHB, LUBZ]</td>\n",
       "      <td>{MOSK2, JEDR, RYNR, TRZS, NWLU, GUZI, PPOL, RU...</td>\n",
       "      <td>../data/igf/mseeds/mseeds_2020/20200123060736....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     stations_with_missing_traces   \n",
       "path                                                \n",
       "20200109224901.mseed                       [RYNR]  \\\n",
       "20200111165827.mseed                       [TRN2]   \n",
       "20200112055037.mseed                       [TRZS]   \n",
       "20200114011249.mseed                       [PCHB]   \n",
       "20200123060736.mseed                       [GROD]   \n",
       "\n",
       "                                      stations_with_missing_annotations   \n",
       "path                                                                      \n",
       "20200109224901.mseed  [RZEC, TRBC2, LUBW, TRZS, PPOL, OBIS, GROD, RU...  \\\n",
       "20200111165827.mseed        [RZEC, TRBC2, LUBW, TRZS, OBIS, RUDN, LUBZ]   \n",
       "20200112055037.mseed  [RZEC, TRBC2, MOSK2, LUBW, BRDW, SGOR, GUZI, P...   \n",
       "20200114011249.mseed  [RZEC, TRBC2, LUBW, DWOL, BRDW, TRZS, SGOR, GU...   \n",
       "20200123060736.mseed        [RZEC, TRBC2, LUBW, BRDW, OBIS, PCHB, LUBZ]   \n",
       "\n",
       "                                           stations_with_available_data   \n",
       "path                                                                      \n",
       "20200109224901.mseed  {MOSK2, JEDR, DWOL, BRDW, GUZI, NWLU, TRN2, PC...  \\\n",
       "20200111165827.mseed  {MOSK2, JEDR, DWOL, RYNR, BRDW, GUZI, NWLU, PP...   \n",
       "20200112055037.mseed   {JEDR, DWOL, RYNR, NWLU, RUDN, TRN2, GROD, ZMST}   \n",
       "20200114011249.mseed  {MOSK2, JEDR, RYNR, PPOL, RUDN, TRN2, GROD, ZMST}   \n",
       "20200123060736.mseed  {MOSK2, JEDR, RYNR, TRZS, NWLU, GUZI, PPOL, RU...   \n",
       "\n",
       "                                                                   path  \n",
       "path                                                                     \n",
       "20200109224901.mseed  ../data/igf/mseeds/mseeds_2020/20200109224901....  \n",
       "20200111165827.mseed  ../data/igf/mseeds/mseeds_2020/20200111165827....  \n",
       "20200112055037.mseed  ../data/igf/mseeds/mseeds_2020/20200112055037....  \n",
       "20200114011249.mseed  ../data/igf/mseeds/mseeds_2020/20200114011249....  \n",
       "20200123060736.mseed  ../data/igf/mseeds/mseeds_2020/20200123060736....  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_data[~missing_data.stations_with_missing_traces.isna()].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7c2b99e9-9428-460c-87e9-c1704a701bd8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stations_with_missing_traces</th>\n",
       "      <th>stations_with_missing_annotations</th>\n",
       "      <th>stations_with_available_data</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>path</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20200101100941.mseed</th>\n",
       "      <td>None</td>\n",
       "      <td>[OBIS, LUBZ]</td>\n",
       "      <td>{RZEC, TRBC2, MOSK2, JEDR, RYNR, LUBW, BRDW, T...</td>\n",
       "      <td>../data/igf/mseeds/mseeds_2020/20200101100941....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20200102080305.mseed</th>\n",
       "      <td>None</td>\n",
       "      <td>[RZEC, LUBW, RYNR, TRZS, SGOR, OBIS, TRN2, PCH...</td>\n",
       "      <td>{TRBC2, MOSK2, JEDR, BRDW, NWLU, GUZI, PPOL, Z...</td>\n",
       "      <td>../data/igf/mseeds/mseeds_2020/20200102080305....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20200103143651.mseed</th>\n",
       "      <td>None</td>\n",
       "      <td>[RZEC, TRBC2, MOSK2, LUBW, BRDW, TRZS, SGOR, G...</td>\n",
       "      <td>{JEDR, DWOL, RYNR, TRN2, GROD, RUDN}</td>\n",
       "      <td>../data/igf/mseeds/mseeds_2020/20200103143651....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20200104050102.mseed</th>\n",
       "      <td>None</td>\n",
       "      <td>[RZEC, TRBC2, LUBW, JEDR, RYNR, BRDW, TRZS, NW...</td>\n",
       "      <td>{MOSK2, DWOL, GUZI, GROD, ZMST, SGOR}</td>\n",
       "      <td>../data/igf/mseeds/mseeds_2020/20200104050102....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20200104145207.mseed</th>\n",
       "      <td>None</td>\n",
       "      <td>[RZEC, TRBC2, LUBW, TRZS, NWLU, OBIS, PCHB, GR...</td>\n",
       "      <td>{MOSK2, JEDR, DWOL, RYNR, BRDW, GUZI, PPOL, TR...</td>\n",
       "      <td>../data/igf/mseeds/mseeds_2020/20200104145207....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     stations_with_missing_traces   \n",
       "path                                                \n",
       "20200101100941.mseed                         None  \\\n",
       "20200102080305.mseed                         None   \n",
       "20200103143651.mseed                         None   \n",
       "20200104050102.mseed                         None   \n",
       "20200104145207.mseed                         None   \n",
       "\n",
       "                                      stations_with_missing_annotations   \n",
       "path                                                                      \n",
       "20200101100941.mseed                                       [OBIS, LUBZ]  \\\n",
       "20200102080305.mseed  [RZEC, LUBW, RYNR, TRZS, SGOR, OBIS, TRN2, PCH...   \n",
       "20200103143651.mseed  [RZEC, TRBC2, MOSK2, LUBW, BRDW, TRZS, SGOR, G...   \n",
       "20200104050102.mseed  [RZEC, TRBC2, LUBW, JEDR, RYNR, BRDW, TRZS, NW...   \n",
       "20200104145207.mseed  [RZEC, TRBC2, LUBW, TRZS, NWLU, OBIS, PCHB, GR...   \n",
       "\n",
       "                                           stations_with_available_data   \n",
       "path                                                                      \n",
       "20200101100941.mseed  {RZEC, TRBC2, MOSK2, JEDR, RYNR, LUBW, BRDW, T...  \\\n",
       "20200102080305.mseed  {TRBC2, MOSK2, JEDR, BRDW, NWLU, GUZI, PPOL, Z...   \n",
       "20200103143651.mseed               {JEDR, DWOL, RYNR, TRN2, GROD, RUDN}   \n",
       "20200104050102.mseed              {MOSK2, DWOL, GUZI, GROD, ZMST, SGOR}   \n",
       "20200104145207.mseed  {MOSK2, JEDR, DWOL, RYNR, BRDW, GUZI, PPOL, TR...   \n",
       "\n",
       "                                                                   path  \n",
       "path                                                                     \n",
       "20200101100941.mseed  ../data/igf/mseeds/mseeds_2020/20200101100941....  \n",
       "20200102080305.mseed  ../data/igf/mseeds/mseeds_2020/20200102080305....  \n",
       "20200103143651.mseed  ../data/igf/mseeds/mseeds_2020/20200103143651....  \n",
       "20200104050102.mseed  ../data/igf/mseeds/mseeds_2020/20200104050102....  \n",
       "20200104145207.mseed  ../data/igf/mseeds/mseeds_2020/20200104145207....  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_data[~missing_data.stations_with_missing_annotations.isna()].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8acde36d-dbd6-4ddc-95fd-085a3dd14be4",
   "metadata": {},
   "source": [
    "### Inspect cases with missing annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4c7b36d1-441b-427e-9485-900172e086d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stations with missing annotations:  ['OBIS', 'LUBZ']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "57 Trace(s) in Stream:\n",
       "\n",
       "PL.BRDW..EHE | 2020-01-01T10:09:36.480000Z - 2020-01-01T10:09:56.480000Z | 100.0 Hz, 2001 samples\n",
       "...\n",
       "(55 other traces)\n",
       "...\n",
       "PL.ZMST..EHZ | 2020-01-01T10:09:36.480000Z - 2020-01-01T10:09:56.480000Z | 100.0 Hz, 2001 samples\n",
       "\n",
       "[Use \"print(Stream.__str__(extended=True))\" to print all Traces]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fname = missing_data[~missing_data.stations_with_missing_annotations.isna()].index[0]\n",
    "missing_stations = missing_data.loc[fname].stations_with_missing_annotations\n",
    "print(\"Stations with missing annotations: \", missing_stations)\n",
    "st = read(missing_data.loc[fname].path)\n",
    "st"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "23819d30-bc55-4263-b374-bb6541bc4ac9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_traces_from_mseed(phases, mseed_fname, path=data_path+\"mseeds/mseeds_2020\", channel=None): \n",
    "\n",
    "    phases = [ann.split(' ') for ann in phases.split(', ')]\n",
    "    phases = pd.DataFrame(phases, columns = ['phase', 'station', 'date', 'hour'])\n",
    "    phases.index = pd.DatetimeIndex(phases.date + ' ' + phases.hour, tz= \"UTC\")\n",
    "\n",
    "    stream = read(f\"{path}/{mseed_fname}\")\n",
    "    traces_count = len(stream.traces)\n",
    "    if channel is not None: \n",
    "        traces_count = int(traces_count/3) \n",
    "        \n",
    "    fig, axes = plt.subplots(traces_count, figsize=(20, traces_count*1.5))\n",
    "    \n",
    "    for i, trace in enumerate(stream.traces): \n",
    "        trace_series = pd.Series(trace.data)\n",
    "        \n",
    "        if channel is not None: \n",
    "            idx = int(i/3)\n",
    "            if trace.stats.channel[-1]!=channel: \n",
    "                continue\n",
    "        else: \n",
    "            idx = i\n",
    "            \n",
    "        freq = 1000 / trace.stats.sampling_rate\n",
    "        trace_series.index = pd.date_range(start=str(trace.stats.starttime), end=str(trace.stats.endtime), freq=f'{freq}ms')\n",
    "\n",
    "        trace_series.plot(title=f\"{trace.stats.station} {trace.stats.channel}\", ax=axes[idx])\n",
    "\n",
    "        station = trace.stats.station\n",
    "        station_annotations = phases[phases.station==station]\n",
    "        p_timestamps = station_annotations[station_annotations.phase=='Pg'].index\n",
    "        s_timestamps = station_annotations[station_annotations.phase=='Sg'].index\n",
    "\n",
    "        if len(p_timestamps) == 0: \n",
    "            print(\"No P annotations for station\", station, trace.stats.starttime)\n",
    "        if len(s_timestamps) == 0: \n",
    "            print(\"No S annotations for station\", station, trace.stats.starttime)\n",
    "\n",
    "        for pt in p_timestamps: \n",
    "            if pt < trace_series.index[0] or pt > trace_series.index[-1]:\n",
    "                print(f\"Pg {pt} out of scope for trace {trace}\")\n",
    "            else: \n",
    "                axes[idx].axvline(x=pt, color='blue', linestyle='--', label='P')\n",
    "\n",
    "        for st in s_timestamps: \n",
    "            if st < trace_series.index[0] or st > trace_series.index[-1]:\n",
    "                print(f\"Sg {st} out of scope for trace {trace}\")\n",
    "            else:\n",
    "                axes[idx].axvline(x=st, color='orange', linestyle='--', label='S')\n",
    "                \n",
    "        axes[idx].legend(loc='upper left')\n",
    "\n",
    "    plt.savefig(f'{data_path}annotated_traces/{mseed_fname}.pdf')\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "71c4c255-a2ba-42f1-9b63-b349ecb7fb1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n",
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No P annotations for station LUBZ 2020-01-01T10:09:36.480000Z\n",
      "No S annotations for station LUBZ 2020-01-01T10:09:36.480000Z\n",
      "No P annotations for station OBIS 2020-01-01T10:09:36.480000Z\n",
      "No S annotations for station OBIS 2020-01-01T10:09:36.480000Z\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 1281508 characters]",
      "text/plain": [
       "<Figure size 2000x2850 with 19 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "phases = data[data.mseed_name == fname].Phases[0]\n",
    "plot_traces_from_mseed(phases, fname, channel='Z')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cce54bb-c15b-4e64-826e-be1604c56de5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "4f7e1a21-bded-43db-b787-c765d6dacc0b",
   "metadata": {},
   "source": [
    "## Plot statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5f84b79-d088-4a6a-842f-c4f48be95713",
   "metadata": {},
   "source": [
    "### Annotations per station"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "29900509-0657-43cb-bf7c-be81a7a9a696",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: title={'center': 'Phases occurences per station'}>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 47936 characters]",
      "text/plain": [
       "<Figure size 1500x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "stats.transpose().plot(kind='bar', title = 'Phases occurences per station', figsize=(15,5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a6049a83-cbba-42ef-b756-3a83f4511bb9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18002.0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats.loc['p'].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71873d25-1e39-4392-bc85-85beece78dfa",
   "metadata": {},
   "source": [
    "### Missing annotations per station"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b0f5988a-f6db-4319-8e0a-c780d7106f0c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: title={'center': 'Missing annotations per station'}>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 40456 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "missing_station_list = []\n",
    "missing_station_list.extend(st[0] if st is not None else None for st in missing_data.stations_with_missing_annotations.values)\n",
    "pd.Series(missing_station_list).value_counts().plot(kind='barh', title='Missing annotations per station')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a58fa9d0-3078-4bc3-bdde-0b456c2f7273",
   "metadata": {},
   "source": [
    "### Missing traces per station"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "79f22b4f-530b-47a0-9247-310384d7181c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: title={'center': 'Missing traces'}>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 27092 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "missing_traces_list = []\n",
    "missing_traces_list.extend(st[0] if st is not None else None for st in missing_data.stations_with_missing_traces.values)\n",
    "pd.Series(missing_traces_list).value_counts().plot(kind='barh', title='Missing traces')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9082b5fd-2789-469c-82aa-29cbbbe654e5",
   "metadata": {},
   "source": [
    "### Annotations over time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fec57a0e-6455-47b1-91be-016c20cb8e6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.merge(data, missing_data, left_on='mseed_name', right_index=True, how='outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "68195c50-81fa-495a-86ce-ecfed4a068ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: title={'center': 'Cumulative number of traces with Pg annotations'}, xlabel='Datetime'>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 44248 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data['Pg_count'] = data.stations_with_available_data.apply(lambda x: len(x) if not pd.isna(x) else 0)\n",
    "data['Pg_count_cumsum']= data.Pg_count.cumsum()\n",
    "data.index = data.Datetime\n",
    "data.Pg_count_cumsum.plot(title=\"Cumulative number of traces with Pg annotations\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b8b3525e-5133-43fd-bba4-829c7bf3004f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(data.stations_with_missing_annotations.values[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "6ae59f8b-4971-42a3-8b04-7a107085f7d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['missing_annotations'] = data.stations_with_missing_annotations.apply(lambda x: len(x) if type(x)==list else 0)\n",
    "data['missing_traces'] = data.stations_with_missing_traces.apply(lambda x: len(x) if type(x)==list else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f78fb182-6155-4a3c-8ccc-c3595dbf3c7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 55276 characters]",
      "text/plain": [
       "<Figure size 1500x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "ax = data.groupby([pd.Grouper(key='Datetime', freq='M')])['missing_annotations'].sum().plot(kind='bar', \n",
    "                                                                                            title=\"Missing annotations per month\",\n",
    "                                                                                            rot=45, \n",
    "                                                                                           figsize=(15,5))\n",
    "  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "45d9e44d-24cf-4b29-b78e-74573731c91d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: title={'center': 'Missing traces in mseeds over time'}, xlabel='Datetime'>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 33056 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.missing_traces.plot(marker='.', linestyle='none', title='Missing traces in mseeds over time')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "265a514b-e0b6-4d3c-9087-cbe418d7c4a7",
   "metadata": {},
   "source": [
    "# Plot traces and save figs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "9501545c-abd9-42c5-b515-ccc6479e94ea",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n",
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n",
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No P annotations for station LUBZ 2020-01-01T10:09:36.480000Z\n",
      "No S annotations for station LUBZ 2020-01-01T10:09:36.480000Z\n",
      "No P annotations for station LUBZ 2020-01-01T10:09:36.480000Z\n",
      "No S annotations for station LUBZ 2020-01-01T10:09:36.480000Z\n",
      "No P annotations for station LUBZ 2020-01-01T10:09:36.480000Z\n",
      "No S annotations for station LUBZ 2020-01-01T10:09:36.480000Z\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n",
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n",
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No P annotations for station OBIS 2020-01-01T10:09:36.478000Z\n",
      "No S annotations for station OBIS 2020-01-01T10:09:36.478000Z\n",
      "No P annotations for station OBIS 2020-01-01T10:09:36.476000Z\n",
      "No S annotations for station OBIS 2020-01-01T10:09:36.476000Z\n",
      "No P annotations for station OBIS 2020-01-01T10:09:36.480000Z\n",
      "No S annotations for station OBIS 2020-01-01T10:09:36.480000Z\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 3595632 characters]",
      "text/plain": [
       "<Figure size 2000x8550 with 57 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "for phases, fname in zip(data.Phases, data.mseed_name): \n",
    "    if not pd.isna(fname): \n",
    "        plot_traces_from_mseed(phases, fname)\n",
    "        break\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a2f222f4-f7d7-4427-bb3a-15af7b75dad0",
   "metadata": {},
   "outputs": [],
   "source": [
    "with pd.ExcelWriter(data_path+'catalog with missing data.xlsx') as writer:  \n",
    "    data.to_excel(writer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4763f667-476f-4029-a392-b4d5093840d4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
