# SPDX-License-Identifier: MIT
# SPDX-FileCopyrightText: Copyright (c) 2026 TU Wien & AWST
# SPDX-FileCopyrightText: For a full list of authors, see the AUTHORS file.
import glob
import os
import numpy as np
import pandas as pd
import yaml
from typing import Union
import xarray as xr
from pathlib import Path
from qa4sm_autoreports.run import (ValidationRun, ValidationConfiguration)
"""
Containers for various types of data from QA4SM (e.g. stats, results, config,
...). Each container comes with a function to collect the required data.
"""
[docs]
def load_yml_to_dict(filepath: str | Path) -> dict:
"""
Load a QA4SM-style YAML config/results file into a nested dictionary.
Parameters
----------
filepath: str or Path
Path to the yml content
Returns:
-------
data: dict
The first level are the content sections names, sub-levels are the
variables in that section.
"""
filepath = Path(filepath)
if not filepath.exists():
raise FileNotFoundError(f"No file found at: {filepath}")
if filepath.suffix not in (".yml", ".yaml"):
raise ValueError(f"Expected a .yml/.yaml file, got: {filepath.suffix}")
with open(filepath, "r", encoding="utf-8") as f:
data = yaml.safe_load(f)
if not isinstance(data, dict):
raise ValueError("YAML content did not parse to a dictionary.")
return data
[docs]
class Data:
"""
Data container base class for variables from various sources
"""
def __init__(self, data=None):
if data is None:
self.data = self._reset()
else:
self.data = data
def _reset(self):
# Empty data dict
self.data = dict()
return self.data
[docs]
@classmethod
def from_yml(cls, path: Union[Path, str]):
"""
Load data from a previous stored yml file
Parameters
----------
path: str or Path
Path to the yml file to load content from
"""
data = load_yml_to_dict(path)
return cls(data=data)
[docs]
def load(self, path, mode='r'):
"""
Load data from the passed yml file into this Data
container.
Parameters
----------
path: str
Path to the yml file to take content from
mode: str, optional
'r': Read mode will drop any already loaded content
'a' Append mode will add content from the to anything loaded
"""
data = load_yml_to_dict(path)
if mode == 'a':
self.data.update(data)
elif mode == 'r':
self.data = data
[docs]
def append(self, other):
""" Add data from other variable """
self.data.update(other.data)
[docs]
def add(self, content: dict, section: str = "Content"):
"""
Add content to the collection.
Parameters
----------
content: dict
Yaml storable content, usually {KEY: Value, ...}
section: str
Multiple contents can be stored, specify name of content
group (e.g. summary statistics).
Each name will be a separate yml section upon export.
"""
self.data[section] = content
[docs]
def dump(self, path: Union[Path, str], overwrite: bool = False):
"""
Write content to yml to import later for the report.
Parameters
----------
path: str
File path to write data to
overwrite: bool, optional
Overwrite will replace an existing file, otherwise will append
to it.
"""
path = Path(path)
os.makedirs(path.parent, exist_ok=True)
if os.path.exists(path) and (not overwrite):
mode = 'a'
else:
mode = 'w'
with open(str(path), mode) as f:
yaml.dump(self.data, f, default_flow_style=False, sort_keys=False)
[docs]
class RunData(Data):
"""
Collection of data from multiple validation runs for a report
"""
def __init__(self, validation_run: ValidationRun):
self.run = validation_run
super(RunData, self).__init__()
[docs]
class RemoteData(RunData):
"""
Collect variables from service API sources
"""
def __init__(self, validation_run: ValidationRun):
super().__init__(validation_run)
[docs]
def collect(self):
"""
Collect Configuration Variables from all datasets in this validation
run.
Returns:
--------
run_vars: dict
Collection of config variables from this run
"""
status, progress = self.run.status
time = self.run.timing()
start = str(time['start']) or None
end = str(time['end']) or None
run_vars = {
'run_status': status,
'run_progress': progress,
'run_duration': time['duration'],
'run_start': start,
'run_end': end
}
self.add(run_vars, "RemoteVars")
return self
[docs]
class SummaryStatsData(RunData):
"""
Collect variables from the csv summary stats file of the validation run
"""
def __init__(self, validation_run: ValidationRun):
self.unit_sep = " in"
super().__init__(validation_run)
def _load_sum_stats(self, drop_unit=False,
keep_dataset=False) -> pd.DataFrame:
df = pd.read_csv(
self.run.local_root / 'summary_stats.csv', index_col=0)
# get rid of unit from table:
if drop_unit:
new_index = [
i.split(self.unit_sep)[0].strip() for i in df.index.values
]
df.index = new_index
df.index = [i.strip() for i in df.index]
if ("Dataset" in df.columns) and not keep_dataset:
df = df.drop(columns="Dataset")
if "Spearman's ρ" in df.index:
df = df.rename(index={"Spearman's ρ": "Spearman's rho"})
if "# observations" in df.index:
df = df.rename(index={"# observations": "N. Obs."})
return df
[docs]
def export_table(self, path=None):
"""
Export the data to a csv table that is used in the latex report
Parameters
----------
path: str
Path to csv file
"""
df = self._load_sum_stats()
df.to_csv(path, sep=';', index_label='Metric')
[docs]
def collect(self):
"""
Collect all relevant stats from the downloaded summary table
"""
df = self._load_sum_stats(drop_unit=True, keep_dataset=True)
datasets = df.pop("Dataset") if "Dataset" in df.columns else None
# Runs with multiple candidate datasets have one row per metric and
# dataset. Then the dataset is added to the variable name, e.g.
# MEAN_PEARSON'S R_1-C3S_COMBINED (version in brackets removed).
duplicated = df.index.duplicated(keep=False)
params = {}
for n, i in enumerate(df.index.values):
name = i.split(self.unit_sep)[0]
if duplicated[n] and (datasets is not None):
name += "_" + str(datasets.iloc[n]).split('(')[0].strip()
for c in df.columns.values:
params[f"{c.upper()}_{name.upper()}"] = float(df.iloc[n][c])
self.add(params, "SummaryVars")
return self
[docs]
class NetcdfData(RunData):
"""
Collect variables from the results netcdf file
"""
def __init__(self, validation_run: ValidationRun):
super().__init__(validation_run)
[docs]
def collect_content(self) -> dict:
ds = xr.open_dataset(self.run.local_root / f"{self.run.remote_id}.nc")
n_gpis = len(ds["gpi"].values)
status_code_ok = 0
n_status_points = []
n_status_ok = []
for var in list(ds.variables.keys()):
if var.startswith("status_"):
n_status_points.append(int(len(ds[var].values.flatten())))
n_status_ok.append(
int(np.sum(ds[var].values.flatten() == status_code_ok)))
data = {
'n_gpis':
int(n_gpis),
'n_status':
n_status_points,
'n_ok':
n_status_ok,
'n_failed': [
int(s - o) for o, s in zip(n_status_ok, n_status_points)
],
'percent_ok': [
int((o / s) * 100)
for o, s in zip(n_status_ok, n_status_points)
]
}
if 'network' in ds.variables:
data['networks'] = self._network_stats(ds)
# The same per dataset pair, e.g. '0-ISMN_and_1-C3S_combined', for
# runs with multiple candidate datasets
pairs = {}
for var in list(ds.variables.keys()):
if not var.startswith("status_between_"):
continue
pair = var[len("status_between_"):]
vals = ds[var].values.flatten()
n, ok = int(len(vals)), int(np.sum(vals == status_code_ok))
pairs[pair] = {
'n_status': n,
'n_ok': ok,
'n_failed': n - ok,
'percent_ok': int((ok / n) * 100) if n > 0 else 0,
}
if 'network' in ds.variables:
pairs[pair]['networks'] = self._network_stats(ds, pair)
data['pairs'] = pairs
return data
@staticmethod
def _first_var(ds: xr.Dataset, prefix: str) -> Union[str, None]:
# First variable in the dataset whose name starts with prefix
for var in ds.data_vars:
if str(var).startswith(prefix):
return str(var)
return None
def _network_stats(self, ds: xr.Dataset, pair=None) -> list:
"""
Per-network summary for in situ (ISMN) validation runs: number of
sensors, successful validations and median R / ubRMSD.
The table is also exported to latex/network_stats.csv
Parameters
----------
ds: xr.Dataset
Validation results
pair: str, optional
Dataset pair, e.g. '0-ISMN_and_1-C3S_combined', to use the
results of. By default the first pair in the results is used.
The table is then exported to latex/network_stats_<pair>.csv
"""
if 'tsw' in ds.dims:
# one value per sensor: use the bulk (full period) results
ds = ds.sel(tsw='bulk') if 'bulk' in ds['tsw'].values \
else ds.isel(tsw=0)
def find(prefix):
if pair is None:
return self._first_var(ds, prefix)
var = f"{prefix}{pair}"
return var if var in ds.variables else None
status_var = find('status_between_')
cols = {'network': ds['network'].values}
if status_var is not None:
cols['ok'] = ds[status_var].values.flatten() == 0
for name, prefix in [('R', 'R_between_'), ('urmsd', 'urmsd_between_')]:
var = find(prefix)
if var is not None:
cols[name] = ds[var].values.flatten()
df = pd.DataFrame(cols)
df['network'] = df['network'].astype(str)
agg = {'n_sensors': ('network', 'size')}
if 'ok' in df:
agg['n_ok'] = ('ok', 'sum')
for name in ['R', 'urmsd']:
if name in df:
agg[f'median_{name}'] = (name, 'median')
stats = df.groupby('network').agg(**agg).reset_index()
stats = stats.sort_values('n_sensors', ascending=False)
os.makedirs(self.run.local_root / 'latex', exist_ok=True)
fname = 'network_stats.csv' if pair is None \
else f'network_stats_{pair}.csv'
stats.to_csv(self.run.local_root / 'latex' / fname,
sep=';', index=False, float_format="%.2f", na_rep='--')
records = []
for rec in stats.to_dict(orient='records'):
records.append({
k: (None if (isinstance(v, float) and pd.isna(v)) else
(v if isinstance(v, str) else
(int(v) if k.startswith('n_') else float(v))))
for k, v in rec.items()
})
return records
[docs]
def collect(self):
nc_attrs = self.collect_content()
self.add(nc_attrs, "NetcdfVars")
return self
[docs]
class ConfigData(RunData):
"""
Collect variables from the validation run config
"""
def __init__(self, validation_run: ValidationRun):
super().__init__(validation_run)
c = glob.glob(os.path.join(self.run.local_root, f"config-*.json"))[0]
self.config_path = c
self.conf = ValidationConfiguration.from_file(self.config_path)
[docs]
def collect_datasets(self, i=0):
"""
Collect the Variables from a dataset configuration
Parameters
----------
i: int, optional
Id of the dataset to read from the config
"""
dataset_config = self.conf['dataset_configs'][i].copy()
id = dataset_config['dataset_id']
dataset_config[f'name'] = (
self.run.connection.dataset_info(id))['pretty_name']
id = dataset_config['version_id']
dataset_config[f'version'] = (
self.run.connection.version_info(id))['pretty_name']
id = dataset_config['variable_id']
dataset_config[f'variable'] = (
self.run.connection.variable_info(id))['pretty_name']
active_filters = []
filters = dataset_config['basic_filters']
for filter in filters:
finf = self.run.connection.filter_info(filter)
active_filters.append({
'description': finf['description'],
'help_text': finf['help_text']
})
dataset_config['basic_filters_description'] = active_filters
# todo: How do param filters work?
active_filters = []
filters = dataset_config['parametrised_filters']
for filter in filters:
filter_id = filter['id']
finf = self.run.connection.filter_info(filter_id)
active_filters.append({
'description': finf['description'],
'help_text': finf['help_text'],
'parameters': filter['parameters']
})
dataset_config['parametrised_filters_description'] = active_filters
return dataset_config
[docs]
def collect(self):
"""
Collect Configuration Variables from all datasets in this validation
run.
Returns:
--------
run_vars: dict
Collection of config variables from this run
"""
run_vars = {
'scaling_reference': None,
'spatial_reference': None,
'temporal_reference': None,
'tcol_metrics': None,
'stability_metrics': None,
'interval_days': None,
}
for id in range(len(self.conf['dataset_configs'])):
ds_vars = self.collect_datasets(id)
if bool(ds_vars['is_spatial_reference']):
run_vars['spatial_reference'] = \
f"{ds_vars['name']} ({ds_vars['version']})"
if bool(ds_vars['is_temporal_reference']):
run_vars['temporal_reference'] = \
f"{ds_vars['name']} ({ds_vars['version']})"
if bool(ds_vars['is_scaling_reference']):
run_vars['scaling_reference'] = \
f"{ds_vars['name']} ({ds_vars['version']})"
run_vars[f'DS{id}'] = ds_vars
_d = self.conf.data.copy()
for m in _d['metrics']:
if m['id'] == 'tcol':
run_vars["tcol_metrics"] = m['value']
if m['id'] == 'stability_metrics':
run_vars["stability_metrics"] = m['value']
interval_days = (pd.to_datetime(_d['interval_to']) -
pd.to_datetime(_d['interval_from'])).days
run_vars["interval_days"] = interval_days
_ = _d.pop("dataset_configs")
run_vars.update(_d)
self.add(run_vars, "ConfigVars")
return self