Illustrating Badgers for testing ML predictors¶
End-to-end walk-through of the paper experiment, step by step:
- Data —
make_data(): four corner blobs with columns(x, y, cls) - Split — stratified 50/50 train/test (
split_data) - Direct (non-causal) deficits — generators applied to the test half: drift, noise, outliers
- Causal deficits — the same deficits injected through the SCM fitted on the train half (
build_dgp), so a \(do\)-intervention on \(x\) propagates to \(y\) - Predictors + impact table — RandomForestClassifier / RandomForestRegressor / IsolationForest fitted once on the clean train half, evaluated on every deficit variant
- Decision maps and regression curves — frozen classifier and detector maps over clean/direct/causal data, plus class-conditioned Random Forest regression curves for predicting \(y\) from \((x, cls)\)
Requires the badgers package and the causal generators' optional dowhy extra (pip install badgers[dowhy]).
0. Setup¶
from pathlib import Path
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.colors import Normalize, to_rgb
from matplotlib.cm import ScalarMappable
from matplotlib.lines import Line2D
from sklearn.inspection import DecisionBoundaryDisplay
from numpy.random import default_rng
SEED = 42
# Okabe-Ito palette (color-vision-deficiency safe) + B/W-distinguishable markers
CLASS_COLORS = ["#0072B2", "#E69F00", "#009E73", "#D55E00"]
CLASS_MARKERS = ["o", "s", "^", "D"]
plt.rcParams.update({
"font.family": "serif", "font.serif": "DejaVu Serif",
"mathtext.fontset": "cm",
"font.size": 8, "axes.titlesize": 8, "axes.labelsize": 8,
"xtick.labelsize": 7, "ytick.labelsize": 7, "legend.fontsize": 6.5,
"axes.linewidth": 0.6, "xtick.major.size": 2.0,
"ytick.major.size": 2.0, "xtick.major.width": 0.6,
"ytick.major.width": 0.6,
# display/export resolution: all point-based sizes (fonts, markers,
# linewidths) scale with dpi, so proportions stay unchanged; PNG
# exports use savefig.dpi, PDF export is vector regardless
"figure.dpi": 200, "savefig.dpi": 300,
})
# export helper for the figures below: save_figure(fig, "name") writes
# a vector PDF and a high-resolution PNG into generated-data/figures/
# (git-ignored, regenerated on every run)
from pathlib import Path
FIG_DIR = Path("generated-data") / "figures"
FIG_DIR.mkdir(parents=True, exist_ok=True)
def save_figure(fig, stem):
"""Write figures/<stem>.pdf (vector) and <stem>.png (savefig.dpi)."""
fig.savefig(FIG_DIR / f"{stem}.pdf")
fig.savefig(FIG_DIR / f"{stem}.png")
return fig
Visualization helpers¶
def scatter_classes(ax, X, y, *, observed=None, ms=3.5, edgecolor=None,
mew=None, color_fn=None, alpha=1.0):
observed = np.ones(len(X), dtype=bool) if observed is None else observed
for cls, (color, marker) in enumerate(zip(CLASS_COLORS, CLASS_MARKERS)):
ix = (y == cls) & observed
ax.plot(X[ix, 0], X[ix, 1], marker,
color=color if color_fn is None else color_fn(color),
markersize=ms, markeredgecolor=edgecolor,
markeredgewidth=0.0 if mew is None else mew, linestyle="none", alpha=alpha)
def caption(ax, text, handle):
ax.legend([handle], [text], loc="lower right", bbox_to_anchor=(1, 0.02),
frameon=False, fontsize=7, labelcolor="0.35",
handlelength=1.0, handletextpad=0.4, borderaxespad=0)
def outlier_marks(ax, X, y, mask, wrong=None, ms=5.0,
show_caption=True,
markeredgewidth=1.0):
"""Appended outlier rows as x-marks, colored by their class; rows in
`wrong` in a darker shade."""
if wrong is not None:
wrong = np.asarray(wrong, dtype=bool)
for cls, color in enumerate(CLASS_COLORS):
m = mask & (y == cls)
if wrong is None:
groups = [(m, color)]
else:
groups = [(m & ~wrong, color), (m & wrong, darken(color))]
for m_, color_ in groups:
if m_.any():
ax.plot(X[m_, 0], X[m_, 1], "x", color=color_, markersize=ms,
markeredgewidth=markeredgewidth,
linestyle="none")
if show_caption:
caption(ax, "outlier rows",
Line2D([], [], marker="x", color="0.35", markersize=ms,
markeredgewidth=markeredgewidth, linestyle="none"))
CLASS_HANDLES = [
Line2D([], [], marker=m, color=c, linestyle="none", markersize=3.5,
label=f"class {i}")
for i, (c, m) in enumerate(zip(CLASS_COLORS, CLASS_MARKERS))
]
def class_legend(fig):
fig.legend(handles=CLASS_HANDLES, loc="outside upper left", ncols=4,
fontsize=8, frameon=False, handlelength=1.0,
handletextpad=0.4)
1. Data generation — four diagonal corner blobs¶
make_databuilds four Gaussian blobs at the corners \((\pm1, \pm1)\) — 400 points total. One seed (SEED = 42) drives a single RNG stream that draws, blob by blob, isotropic noise scaled by the vector \(\mathrm{scales} = (0.5, 0.2)\) and then rotated by \(45°\) via the matrix \(R\). Each blob is elongated along the main diagonal.align_yrebuilds the ground-truth labels for the generated set. The in-place generators return the same 200 rows with the original labels; the outlier generators append 20 rows that are copies of seed rows, so their class is read from the untouchedclscell of the generated matrix (Xt[len(y):, 2]).
def make_data(n_samples=400):
'''
Corner blobs whose per-class scatter is elongated along the main diagonal.
'''
n_per_blob = n_samples // 4
centers = [(-1, -1), (1, -1), (-1, 1), (1, 1)]
rotation = np.array([[np.cos(np.pi / 4), -np.sin(np.pi / 4)],
[np.sin(np.pi / 4), np.cos(np.pi / 4)]])
scales = np.array([0.5, 0.2])
rng = default_rng(SEED)
X = np.vstack([np.asarray(c) + (rng.normal(size=(n_per_blob, 2)) * scales) @ rotation.T
for c in centers])
y = np.repeat(np.arange(4), n_per_blob)
X = np.hstack([X, y.astype(float).reshape(-1, 1)])
return X, y
def align_y(Xt, y, sc):
'''Class labels aligned with the generated output.
Outlier rows are appended copies of seed rows: their cls cell is NOT a
selector target, so it still carries the seed row's class.
'''
if sc.observed is None:
return y
return np.concatenate([y, Xt[len(y):, 2].astype(int)])
Generatring the data¶
Four Gaussian blobs at the corners \((\pm1, \pm1)\), elongated along the main
diagonal (scales \(0.5 \times 0.2\), rotated \(45°\)).
X has three columns
\((x, y, \mathrm{cls})\); y is the class index \(0..3\).
from sklearn.model_selection import train_test_split
X, y = make_data()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, stratify=y, random_state=SEED)
# visualization
fig, axes = plt.subplots(1, 3, figsize=(9.0, 3.8), sharex=True, sharey=True,
constrained_layout=True)
for ax, (label, Xd, yd) in zip(
axes, [("(a) All data, $N=400$", X, y),
("(b) Train, $N=200$", X_train, y_train),
("(c) Test, $N=200$", X_test, y_test)]):
scatter_classes(ax, Xd, yd)
ax.set_title(label)
ax.set_xlim(-3, 4)
ax.set_ylim(-3, 4)
class_legend(fig)
Fitting the causal data-generating process (SCM)¶
build_dgp wraps a structural causal model (SCM) in a badgers
GcmSCMDGP so the causal generators can inject defects and propagate them
through the learned mechanism.
The graph encodes:
- \(\mathrm{cls} \to x\),
- \(\mathrm{cls} \to y\)
- \(x \to y\)
root "cls" gets EmpiricalDistribution — it just resamples observed
class values.
"x" gets an AdditiveNoiseModel whose prediction model is a pipeline
OneHotEncoder → LinearRegression — i.e. the learned conditional mean
\(\mathbb{E}[x \mid \mathrm{cls}]\) (one mean per blob) plus the fitted
noise term, so \(x = f_x(\mathrm{cls}) + N_x\).
"y" gets an AdditiveNoiseModel over the feature block
OneHotEncoder(cls) ⊕ passthrough(x) followed by a linear regression —
i.e. \(y = f_y(\mathrm{cls}, x) + N_y\), capturing both the class offset and
the diagonal tilt. This is the edge that makes a \(do\)-intervention on \(x\)
propagate to \(y\).
GcmSCMDGP(scm, COLUMN_MAPPING).fit(X) maps the column names to positions
(x → 0, y → 1, cls → 2) and fits all noise models on the clean train
half; the resulting dgp object is what the causal generators in section 4
consume. Needs the optional dowhy extra.
# variable name -> column index of the (x, y, cls) design matrix
COLUMN_MAPPING = {"x": 0, "y": 1, "cls": 2}
def build_dgp(X):
'''Fit the shared SCM cls -> x, cls -> y, x -> y on the clean data.'''
import dowhy.gcm as gcm
import networkx as nx
from sklearn.compose import ColumnTransformer
from sklearn.linear_model import LinearRegression
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import OneHotEncoder
from dowhy.gcm.ml.regression import SklearnRegressionModel
from badgers.core.dgp.backends.gcm_scm import GcmSCMDGP
graph = nx.DiGraph([("cls", "x"), ("cls", "y"), ("x", "y")])
scm = gcm.StructuralCausalModel(graph)
scm.set_causal_mechanism("cls", gcm.EmpiricalDistribution())
feature_block = ColumnTransformer(
[("ohe", OneHotEncoder(), [0]), ("keep", "passthrough", [1])])
scm.set_causal_mechanism(
"x", gcm.AdditiveNoiseModel(
prediction_model=SklearnRegressionModel(
make_pipeline(OneHotEncoder(), LinearRegression()))))
scm.set_causal_mechanism(
"y", gcm.AdditiveNoiseModel(
prediction_model=SklearnRegressionModel(
make_pipeline(feature_block, LinearRegression()))))
return GcmSCMDGP(scm, COLUMN_MAPPING).fit(X)
2. Applying the generators¶
Every deficit below is run as a scenario: a name, a family tag
(direct / causal / reference), and the generated test set. Two pieces
of glue are needed around the generators:
Scenariois a small state holder;sc.observedrecords, as a boolean mask aligned with the generated output, which rows are original rows (True) and which are appended outlier rows (False). In-place generators (drift, noise) leave itNone— all rows are original._FrozenSpecis an adapter for the noise generator: instead of a selector object it hands over an already-computed boolean spec of which cells to corrupt, so the noise lands exactly on the cells selected upstream (figure bookkeeping, kept for reproducibility with the paper contract).
class _FrozenSpec:
'''Hands a pre-realized spec to the generator (figure bookkeeping).'''
def __init__(self, spec):
self.spec = spec
def __call__(self, X, targets=None):
return {0: self.spec}
class Scenario:
def __init__(self, name, family):
self.name = name
self.family = family
# bool mask aligned with the generated output; None = all rows are
# original rows (in-place generators)
self.observed = None
The six deficit scenarios¶
Three deficits × two injection layers (plus the untouched baseline), each a function fn(X, y, sc, dgp) -> Xt that applies the exact
generators of the paper figures to the test data:
- direct drift —
FixedShiftClassesGeneratormoves the class-1 rows by the absolute vectorshift={1: (1.0, 0.0)}on the columnsvariables=[0, 1](so \(x \leftarrow x+1\), \(y \leftarrow y\), no propagation);ClassSelectorpicks the affected rows in class-1 proportion. - direct noise —
AdditiveNoiseGeneratoradds \(\mathcal{N}(0, 0.5^2)\) (scale ×noise_std=1.0) to the class-1xcells; the cells are selected up front and passed through_FrozenSpec. - direct outliers —
OutlierSamplingGeneratorappendscount=20rows drawn fromJointHistogramSampler's low-density bins — regions where the seed data has almost no support.sc.observedrecords the 200 original vs 20 appended rows (~gen.get_outliers_labels()). - causal variants — the same three deficits, but as \(do\)-interventions
routed through the fitted SCM (
dgp=dgp):CausalDriftGeneratorshifts \(x\) byshift_amount * std(x),CausalNoiseGeneratoradds relative noisenoise_std=0.5, andCausalOutlierPropagationGeneratorreplaces \(x\) of the 20 seed rows with OOD values and re-samples their descendants (\(y\)) through the SCM. Each causal call re-derives the class-proportion selector, so both layers hit the same rows.
SCENARIOS lists them in the order the impact table consumes; each entry is
instantiated into a fresh Scenario at run time so per-run state
(observed) never leaks between evaluations.
from numpy.random import default_rng
from scipy.stats import norm
from badgers.core.sampling import JointHistogramSampler
from badgers.core.selectors import ClassSelector, get_selected_fields
from badgers.core.selectors.value_free import CountSelector
from badgers.generators.tabular_data.drift import FixedShiftClassesGenerator
from badgers.generators.tabular_data.drift.causal import CausalDriftGenerator
from badgers.generators.tabular_data.noise import AdditiveNoiseGenerator
from badgers.generators.tabular_data.noise.causal import CausalNoiseGenerator
from badgers.generators.tabular_data.outliers.causal import (
CausalOutlierPropagationGenerator,
)
from badgers.generators.tabular_data.outliers.sampling import (
OutlierSamplingGenerator,
)
def sc_baseline(X, y, sc, dgp):
return X.copy()
def sc_drift_direct(X, y, sc, dgp):
gen = FixedShiftClassesGenerator(random_generator=default_rng(0))
return gen.generate(
X.copy(),
selector=ClassSelector([0], class_column=2,
proportions={1: (y == 1).mean()},
rng=default_rng(0)),
shift={1: (1.0, 0.0)}, variables=[0, 1])
def sc_noise_direct(X, y, sc, dgp):
rng = default_rng(42)
gen = AdditiveNoiseGenerator(random_generator=rng)
noisy = get_selected_fields(
X,
ClassSelector([0], class_column=2,
proportions={1: (y == 1).mean()}, rng=rng),
rng,
)[:, 0]
return gen.generate(
X.copy(), distribution=norm(loc=0, scale=0.5), noise_std=1.0,
selector=_FrozenSpec(noisy))
def sc_outliers_direct(X, y, sc, dgp):
gen = OutlierSamplingGenerator(random_generator=default_rng(0))
Xt = gen.generate(
X.copy(),
selector=CountSelector([0], count=20, rng=default_rng(0)),
out_of_distribution_sampler=JointHistogramSampler(
bins=10, threshold_low_density=0.25),
)
sc.observed = ~gen.get_outliers_labels().astype(bool)
return Xt
def sc_drift_causal(X, y, sc, dgp):
gen = CausalDriftGenerator(random_generator=default_rng(10), dgp=dgp)
return gen.generate(
X.copy(),
selector=ClassSelector([COLUMN_MAPPING["x"]],
class_column=COLUMN_MAPPING["cls"],
proportions={1: (y == 1).mean()},
rng=default_rng(0)),
shift_amount=1.0)
def sc_noise_causal(X, y, sc, dgp):
gen = CausalNoiseGenerator(random_generator=default_rng(42), dgp=dgp)
return gen.generate(
X.copy(),
selector=ClassSelector([COLUMN_MAPPING["x"]],
class_column=COLUMN_MAPPING["cls"],
proportions={1: (y == 1).mean()},
rng=default_rng(42)),
noise_std=0.5)
def sc_outliers_causal(X, y, sc, dgp):
gen = CausalOutlierPropagationGenerator(
random_generator=default_rng(0), dgp=dgp)
Xt = gen.generate(
X.copy(),
selector=CountSelector(
[COLUMN_MAPPING["x"]], count=20, rng=default_rng(0)),
out_of_distribution_sampler=JointHistogramSampler(
bins=10, threshold_low_density=0.25),
)
sc.observed = ~gen.get_outliers_labels().astype(bool)
return Xt
SCENARIOS = [
("baseline", "reference", sc_baseline),
("drift", "direct", sc_drift_direct),
("noise", "direct", sc_noise_direct),
("outliers", "direct", sc_outliers_direct),
("drift", "causal", sc_drift_causal),
("noise", "causal", sc_noise_causal),
("outliers", "causal", sc_outliers_causal),
]
Applying direct (non-causal) generators on the test data¶
def to_frame(Xt, sc):
"""Generated test data as an inspectable DataFrame.
One row per generated row: the two feature columns, the true class
(aligned via align_y) and an `outlier` flag for appended rows.
"""
df = pd.DataFrame(Xt[:, :2], columns=["x", "y"])
df["cls"] = align_y(Xt, y_test, sc).astype(int)
if sc.observed is not None:
df["outlier"] = ~sc.observed
return df
def save_frames(frames, tag):
"""Write every scenario DataFrame as one CSV: <tag>-<scenario>.csv."""
out = Path("generated-data")
out.mkdir(exist_ok=True)
for name, df in frames.items():
df.to_csv(out / f"{tag}-{name}.csv", index=False)
return out
# shared panel convention for every 2x2 display below: clean test half
# top-left, drift top-right, noise bottom-left, outliers bottom-right;
# one entry per case (baseline + the six deficit scenarios), keyed like
# PREDICTIONS, so no display cell ever mutates it
CASES = ("test", "drift", "noise", "outliers")
CASE_TITLES = {
"test": "Clean test half, $N=200$",
"direct-drift": r"Drift: $O_X \leftarrow O_X + 1$ for $C = 1$",
"direct-noise": (r"Noise: $O_X \leftarrow O_X + \mathcal{N}(0, 0.5^2)$"
"\n"
r"for $C = 1$"),
"direct-outliers": (r"Outliers: $O_X \leftarrow O_X^{\mathrm{OOD}}$"
"\n"
"low-density histogram sampling"),
"causal-drift": r"Drift: $do(X \leftarrow X + 1)$ for $C = 1$",
"causal-noise": (r"Noise: $do(X \leftarrow X + \mathcal{N}(0, 0.5^2))$"
"\n"
r"for $C = 1$"),
"causal-outliers": (r"Outliers: $do(X \leftarrow X^{\mathrm{OOD}})$"
"\n"
"propagated through the SCM"),
}
# apply the three direct generators to the clean test half and keep
# the results: DIRECT (raw arrays, consumed by the figures below) and
# DIRECT_DATA (same rows as DataFrames, also written to generated-data/)
DIRECT = {}
for name, fn in (("drift", sc_drift_direct), ("noise", sc_noise_direct),
("outliers", sc_outliers_direct)):
sc = Scenario(name, "direct")
DIRECT[name] = (fn(X_test, y_test, sc, None), sc)
DIRECT_DATA = {name: to_frame(Xt, sc) for name, (Xt, sc) in DIRECT.items()}
save_frames(DIRECT_DATA, "direct")
DIRECT_DATA
{'drift': x y cls
0 0.350248 0.544207 3
1 -1.476227 -1.427412 0
2 -1.136071 -0.817239 0
3 -0.504159 1.533031 2
4 -0.645329 -0.600909 0
.. ... ... ...
195 2.060969 -0.809060 1
196 -1.314292 0.434318 2
197 -1.509834 0.406070 2
198 1.778442 -0.738377 1
199 0.729360 0.632531 3
[200 rows x 3 columns],
'noise': x y cls
0 0.350248 0.544207 3
1 -1.476227 -1.427412 0
2 -1.136071 -0.817239 0
3 -0.504159 1.533031 2
4 -0.645329 -0.600909 0
.. ... ... ...
195 0.966667 -0.809060 1
196 -1.314292 0.434318 2
197 -1.509834 0.406070 2
198 0.293139 -0.738377 1
199 0.729360 0.632531 3
[200 rows x 3 columns],
'outliers': x y cls outlier
0 0.350248 0.544207 3 False
1 -1.476227 -1.427412 0 False
2 -1.136071 -0.817239 0 False
3 -0.504159 1.533031 2 False
4 -0.645329 -0.600909 0 False
.. ... ... ... ...
215 -1.723581 -1.487663 1 True
216 0.175774 0.350953 3 True
217 -0.344924 0.399317 3 True
218 0.244976 -1.359137 0 True
219 0.106029 0.434318 2 True
[220 rows x 4 columns]}
Visualizing the generated test data¶
The clean test half (a) next to the three generated variants — the 2x2 grid makes every change directly comparable against the original data.
# 2x2 grid: panel (a) shows the untouched clean test half so every generated
# variant can be compared against the original data at a glance
fig, axes = plt.subplots(2, 2, figsize=(4.5, 4.5), sharex=True, sharey=True,
constrained_layout=True)
ax = axes[0, 0]
scatter_classes(ax, X_test, y_test)
ax.set_title(CASE_TITLES["test"])
for ax, name in zip(axes.flat[1:], CASES[1:]):
Xt, sc = DIRECT[name]
y_al = align_y(Xt, y_test, sc)
if sc.observed is None:
scatter_classes(ax, Xt, y_al)
else:
scatter_classes(ax, Xt, y_al, observed=sc.observed)
outlier_marks(ax, Xt, y_al, ~sc.observed)
ax.set_title(CASE_TITLES[f"direct-{name}"])
for ax in axes.flat:
ax.set_xlim(-3, 4)
ax.set_ylim(-3, 4)
class_legend(fig)
Applying causal generators on the test data¶
The same three deficits, but injected through the SCM
\(\mathrm{cls} \to x\), \(\mathrm{cls} \to y\), \(x \to y\) fitted on the
train half (build_dgp), so a \(do\)-intervention on \(x\) propagates to
\(y\) as well.
# same as the direct cell, but the generators inject the deficits through
# the SCM (dgp) fitted on the clean train half
dgp = build_dgp(X_train) # SCM fitted on the clean train half only
CAUSAL = {}
for name, fn in (("drift", sc_drift_causal), ("noise", sc_noise_causal),
("outliers", sc_outliers_causal)):
sc = Scenario(name, "causal")
CAUSAL[name] = (fn(X_test, y_test, sc, dgp), sc)
CAUSAL_DATA = {name: to_frame(Xt, sc) for name, (Xt, sc) in CAUSAL.items()}
save_frames(CAUSAL_DATA, "causal")
CAUSAL_DATA
c:\Users\siebert\PycharmProjects\badgers\.venv\Lib\site-packages\tqdm\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html from .autonotebook import tqdm as notebook_tqdm Fitting causal mechanism of node y: 100%|██████████| 3/3 [00:00<00:00, 121.99it/s]
{'drift': x y cls
0 0.350248 0.544207 3
1 -1.476227 -1.427412 0
2 -1.136071 -0.817239 0
3 -0.504159 1.533031 2
4 -0.645329 -0.600909 0
.. ... ... ...
195 2.060969 -0.052770 1
196 -1.314292 0.434318 2
197 -1.509834 0.406070 2
198 1.778442 -0.407774 1
199 0.729360 0.632531 3
[200 rows x 3 columns],
'noise': x y cls
0 0.350248 0.544207 3
1 -1.476227 -1.427412 0
2 -1.136071 -0.817239 0
3 -0.504159 1.533031 2
4 -0.645329 -0.600909 0
.. ... ... ...
195 2.132627 -0.745100 1
196 -1.314292 0.434318 2
197 -1.509834 0.406070 2
198 -0.044199 -1.768313 1
199 0.729360 0.632531 3
[200 rows x 3 columns],
'outliers': x y cls outlier
0 0.350248 0.544207 3 False
1 -1.476227 -1.427412 0 False
2 -1.136071 -0.817239 0 False
3 -0.504159 1.533031 2 False
4 -0.645329 -0.600909 0 False
.. ... ... ... ...
215 -1.723581 -3.136745 1 True
216 0.175774 0.110763 3 True
217 -0.344924 -0.489693 3 True
218 0.244976 -0.040717 0 True
219 0.106029 1.763909 2 True
[220 rows x 4 columns]}
# same 2x2 layout as the direct figure: clean test half (a) + causal variants
fig, axes = plt.subplots(2, 2, figsize=(4.5, 4.5), sharex=True, sharey=True,
constrained_layout=True)
ax = axes[0, 0]
scatter_classes(ax, X_test, y_test)
ax.set_title(CASE_TITLES["test"])
for ax, name in zip(axes.flat[1:], CASES[1:]):
Xt, sc = CAUSAL[name]
y_al = align_y(Xt, y_test, sc)
if sc.observed is None:
scatter_classes(ax, Xt, y_al)
else:
scatter_classes(ax, Xt, y_al, observed=sc.observed)
outlier_marks(ax, Xt, y_al, ~sc.observed)
ax.set_title(CASE_TITLES[f"causal-{name}"])
for ax in axes.flat:
ax.set_xlim(-3, 4)
ax.set_ylim(-3, 4)
class_legend(fig)
for name, (Xt, sc) in CAUSAL.items():
print(f"{name:9s} rows: {len(Xt)}")
drift rows: 200 noise rows: 200 outliers rows: 220
3. Training the predictors¶
train_predictors fits the three downstream predictors once, on the clean
train half, and returns them (all with random_state=SEED so the whole
notebook is reproducible):
- clf —
RandomForestClassifier, features \((x, y)\) → class index (theclscolumn cast back to int) - reg —
RandomForestRegressor, features \(x,c\) → x (treating the class index as a continuous blob coordinate) - det —
IsolationForest, unsupervised on \((x, y)\) only, withcontamination=0.1to match the expected outlier rate
from sklearn.ensemble import IsolationForest, RandomForestClassifier, RandomForestRegressor
def train_predictors(X):
clf = RandomForestClassifier(
n_estimators=100, random_state=SEED).fit(X[:, [0,1]], X[:, 2].astype(int))
reg = RandomForestRegressor(
n_estimators=100, random_state=SEED).fit(X[:, [0,2]], X[:, 1])
det = IsolationForest(
n_estimators=100, contamination=0.1, random_state=SEED).fit(X[:, [0,1]])
return {"clf": clf, "reg": reg, "det": det}
predictors = train_predictors(X_train)
4. Measuring how predictors react to the deficits¶
All three predictors are fitted once on the clean train half and then
frozen. Each table row evaluates them on one deficit variant of the test
half (evaluate, labels aligned via align_y: appended outlier rows keep
their seed row's class, read from the untouched cls cell). Columns:
classifier accuracy and macro \(F_1\); regressor MSE and \(R^2\) for the continuous \(y\) target; detector mean anomaly score (higher = more normal), share flagged by
IsolationForest (contamination \(= 0.1\)), and share scoring above the clean
test half's 90th-percentile anomaly score \(p_{90}\).
Evaluation helpers¶
Two helpers make the generated outputs comparable to the clean test half:
evaluatescores the three frozen predictors on the generated setXt = (x, y, cls): classifier accuracy and macro \(F_1\); regressor MSE and \(R^2\) for continuous \(y\); detector mean anomaly score (higher = more normal), share flagged byIsolationForest.predict() == -1, and share scoring above the clean test half's 90th-percentile anomaly scorep90_clean. It returns a flat metric dict for the impact table plus anextradict covering the appended rows only (share flagged / above \(p_{90}\) among the outliers themselves).
from sklearn.metrics import accuracy_score, f1_score, mean_squared_error, r2_score
def evaluate(sc, predictors, Xt, y_full):
'''Metrics on the full generated test set (labels aligned via align_y).'''
y_true = Xt[:, 1]
yp = predictors["clf"].predict(Xt[:, [0,1]])
yrp = predictors["reg"].predict(Xt[:, [0,2]])
scores = predictors["det"].score_samples(Xt[:, [0,1]])
flagged = predictors["det"].predict(Xt[:, [0,1]]) == -1
obs = np.ones(len(Xt), dtype=bool) if sc.observed is None else sc.observed
app = ~obs
m = {
"scenario": sc.name,
"family": sc.family,
"rows": int(len(Xt)),
"clf_acc": accuracy_score(y_full, yp),
"clf_f1m": f1_score(y_full, yp, average="macro"),
"reg_mse": mean_squared_error(y_true, yrp),
"reg_r2": r2_score(y_true, yrp),
"det_score": scores.mean(),
"det_flag%": 100.0 * flagged.mean()
}
extra = {"scenario": f"{sc.name} [{sc.family}]", "family": sc.family,
"rows": len(Xt)}
if app.any():
extra["appended"] = int(app.sum())
extra["app_flag%"] = 100.0 * flagged[app].mean()
return m, extra
rows = []
for name, family, fn in SCENARIOS:
sc = Scenario(name, family)
Xt = fn(X_test, y_test, sc, dgp)
y_full = align_y(Xt, y_test, sc)
m, _ = evaluate(sc, predictors, Xt, y_full)
rows.append(m)
metrics = pd.DataFrame(rows)[
["scenario", "family", "rows", "clf_acc", "clf_f1m",
"reg_mse", "reg_r2", "det_score", "det_flag%"]]
metrics.round(3).set_index(["scenario", "family"]).sort_index().T
| scenario | baseline | drift | noise | outliers | |||
|---|---|---|---|---|---|---|---|
| family | reference | causal | direct | causal | direct | causal | direct |
| rows | 200.000 | 200.000 | 200.000 | 200.000 | 200.000 | 220.000 | 220.000 |
| clf_acc | 0.990 | 0.850 | 0.990 | 0.965 | 0.980 | 0.927 | 0.927 |
| clf_f1m | 0.990 | 0.837 | 0.990 | 0.965 | 0.980 | 0.928 | 0.927 |
| reg_mse | 0.111 | 0.158 | 0.228 | 0.148 | 0.147 | 0.211 | 0.126 |
| reg_r2 | 0.898 | 0.807 | 0.790 | 0.869 | 0.865 | 0.839 | 0.884 |
| det_score | -0.497 | -0.532 | -0.527 | -0.502 | -0.504 | -0.505 | -0.502 |
| det_flag% | 16.000 | 33.500 | 35.000 | 18.500 | 21.000 | 20.909 | 18.182 |
Applying the predictors to the transformed data¶
Idea. The three frozen predictors are applied once to every
transformed test set — the clean test half and the six deficit variants of
the two preceding sections (the same rows that were saved under
generated-data/) — and the predictions are kept in one inspectable data
structure: PREDICTIONS maps a case name to a DataFrame with one row per
generated row:
x,y— observed featurexand continuous regression targety,cls— the true class (aligned viaalign_y) and the regressor's second input,prediction_clf— the classifier's predicted class index,prediction_reg— the regressor's predicted continuousyvalue,prediction_det— the detector's prediction (-1means flagged),outlier—Truefor the appended outlier rows (outlier cases only).
Everything below only reads PREDICTIONS — no predictor is called while
drawing.
def apply_predictors(Xt, y_al, observed, predictors):
"""Apply the three frozen predictors to one generated test set.
Returns an inspectable DataFrame with the observed (x, y, cls) values
and all three frozen predictors' outputs.
"""
df = pd.DataFrame(Xt[:, :2], columns=["x", "y"])
df["cls"] = np.asarray(y_al).astype(int)
df["prediction_clf"] = predictors["clf"].predict(Xt[:, [0, 1]])
df["prediction_reg"] = predictors["reg"].predict(Xt[:, [0, 2]])
df["prediction_det"] = predictors["det"].predict(Xt[:, [0, 1]])
if observed is not None:
df["outlier"] = ~np.asarray(observed, dtype=bool)
return df
# one entry per transformed test set: the clean test half plus the six
# deficit variants of the two preceding sections
PREDICTIONS = {"test": apply_predictors(X_test, y_test, None, predictors)}
for family, scen in (("direct", DIRECT), ("causal", CAUSAL)):
for name, (Xt, sc) in scen.items():
observed = None if sc.observed is None else sc.observed
PREDICTIONS[f"{family}-{name}"] = apply_predictors(
Xt, align_y(Xt, y_test, sc), observed, predictors)
# shared panel convention for the 2x2 displays below: clean test half
# top-left, drift top-right, noise bottom-left, outliers bottom-right; the
# frozen regression grid spans every case shown in the regression panels
x_values = np.concatenate(
[PREDICTIONS[f"{family}-{name}"]["x"].to_numpy() for family in
("direct", "causal") for name in CASES[1:]]
+ [PREDICTIONS["test"]["x"].to_numpy()]
)
X_GRID = np.linspace(x_values.min(), x_values.max(), 500)
PREDICTIONS
{'test': x y cls prediction_clf prediction_reg prediction_det
0 0.350248 0.544207 3 3 0.800141 1
1 -1.476227 -1.427412 0 0 -1.529213 -1
2 -1.136071 -0.817239 0 0 -1.222025 1
3 -0.504159 1.533031 2 2 1.389069 1
4 -0.645329 -0.600909 0 0 -0.656402 1
.. ... ... ... ... ... ...
195 0.952112 -0.809060 1 1 -0.939141 1
196 -1.314292 0.434318 2 2 0.595576 1
197 -1.509834 0.406070 2 2 0.395045 -1
198 0.669585 -0.738377 1 1 -1.060369 1
199 0.729360 0.632531 3 3 0.602403 1
[200 rows x 6 columns],
'direct-drift': x y cls prediction_clf prediction_reg prediction_det
0 0.350248 0.544207 3 3 0.800141 1
1 -1.476227 -1.427412 0 0 -1.529213 -1
2 -1.136071 -0.817239 0 0 -1.222025 1
3 -0.504159 1.533031 2 2 1.389069 1
4 -0.645329 -0.600909 0 0 -0.656402 1
.. ... ... ... ... ... ...
195 2.060969 -0.809060 1 1 -0.262666 -1
196 -1.314292 0.434318 2 2 0.595576 1
197 -1.509834 0.406070 2 2 0.395045 -1
198 1.778442 -0.738377 1 1 -0.473252 -1
199 0.729360 0.632531 3 3 0.602403 1
[200 rows x 6 columns],
'direct-noise': x y cls prediction_clf prediction_reg prediction_det
0 0.350248 0.544207 3 3 0.800141 1
1 -1.476227 -1.427412 0 0 -1.529213 -1
2 -1.136071 -0.817239 0 0 -1.222025 1
3 -0.504159 1.533031 2 2 1.389069 1
4 -0.645329 -0.600909 0 0 -0.656402 1
.. ... ... ... ... ... ...
195 0.966667 -0.809060 1 1 -0.720776 1
196 -1.314292 0.434318 2 2 0.595576 1
197 -1.509834 0.406070 2 2 0.395045 -1
198 0.293139 -0.738377 1 1 -1.669457 1
199 0.729360 0.632531 3 3 0.602403 1
[200 rows x 6 columns],
'direct-outliers': x y cls prediction_clf prediction_reg prediction_det \
0 0.350248 0.544207 3 3 0.800141 1
1 -1.476227 -1.427412 0 0 -1.529213 -1
2 -1.136071 -0.817239 0 0 -1.222025 1
3 -0.504159 1.533031 2 2 1.389069 1
4 -0.645329 -0.600909 0 0 -0.656402 1
.. ... ... ... ... ... ...
215 -1.723581 -1.487663 1 0 -1.623511 -1
216 0.175774 0.350953 3 3 0.712644 1
217 -0.344924 0.399317 3 2 1.119927 1
218 0.244976 -1.359137 0 1 -1.020235 1
219 0.106029 0.434318 2 3 1.183110 1
outlier
0 False
1 False
2 False
3 False
4 False
.. ...
215 True
216 True
217 True
218 True
219 True
[220 rows x 7 columns],
'causal-drift': x y cls prediction_clf prediction_reg prediction_det
0 0.350248 0.544207 3 3 0.800141 1
1 -1.476227 -1.427412 0 0 -1.529213 -1
2 -1.136071 -0.817239 0 0 -1.222025 1
3 -0.504159 1.533031 2 2 1.389069 1
4 -0.645329 -0.600909 0 0 -0.656402 1
.. ... ... ... ... ... ...
195 2.060969 -0.052770 1 3 -0.262666 -1
196 -1.314292 0.434318 2 2 0.595576 1
197 -1.509834 0.406070 2 2 0.395045 -1
198 1.778442 -0.407774 1 1 -0.473252 -1
199 0.729360 0.632531 3 3 0.602403 1
[200 rows x 6 columns],
'causal-noise': x y cls prediction_clf prediction_reg prediction_det
0 0.350248 0.544207 3 3 0.800141 1
1 -1.476227 -1.427412 0 0 -1.529213 -1
2 -1.136071 -0.817239 0 0 -1.222025 1
3 -0.504159 1.533031 2 2 1.389069 1
4 -0.645329 -0.600909 0 0 -0.656402 1
.. ... ... ... ... ... ...
195 2.132627 -0.745100 1 1 -0.262666 -1
196 -1.314292 0.434318 2 2 0.595576 1
197 -1.509834 0.406070 2 2 0.395045 -1
198 -0.044199 -1.768313 1 0 -0.822785 -1
199 0.729360 0.632531 3 3 0.602403 1
[200 rows x 6 columns],
'causal-outliers': x y cls prediction_clf prediction_reg prediction_det \
0 0.350248 0.544207 3 3 0.800141 1
1 -1.476227 -1.427412 0 0 -1.529213 -1
2 -1.136071 -0.817239 0 0 -1.222025 1
3 -0.504159 1.533031 2 2 1.389069 1
4 -0.645329 -0.600909 0 0 -0.656402 1
.. ... ... ... ... ... ...
215 -1.723581 -3.136745 1 0 -1.623511 -1
216 0.175774 0.110763 3 3 0.712644 -1
217 -0.344924 -0.489693 3 0 1.119927 1
218 0.244976 -0.040717 0 3 -1.020235 -1
219 0.106029 1.763909 2 3 1.183110 -1
outlier
0 False
1 False
2 False
3 False
4 False
.. ...
215 True
216 True
217 True
218 True
219 True
[220 rows x 7 columns]}
5. Visualizing predictors decisions boundaries and regression lines¶
Idea. The decision functions are frozen — fitted once on the clean train half — so their boundary images are computed once on dense \(400 \times 400\) grids and reused in the classifier and detector maps below. The classifier colors each grid cell with the winning class, faded toward white by the vote margin (\(\sqrt{\max p}\)). Regression is shown separately as class-conditioned step curves: the model input is \((x, cls)\) and its continuous output is \(y\).
Drawing helpers¶
Idea. All per-point drawing reads the stored predictions, never the
predictors. A row is drawn in a darker shade of its class color when the
panel's predictor handles it wrong — wrong_clf flags rows whose predicted
class differs from the true one. wrong_reg marks rows whose continuous
prediction differs from the observed \(y\) by more than \(0.5\) target units.
overlay draws rows class by class with white rims and stamps appended
outliers with class-colored \(\times\) marks.
# Marker styling for the predictor panels (classifier, detector,
# regression): change sizes and edge widths here.
POINT_MS = 3.4 # observation-point markersize
POINT_EDGE = 0.5 # white rim width around observation points
OUTLIER_MS = 5.0 # appended-outlier x-mark markersize
OUTLIER_EDGE = 1.0 # x-mark stroke width
CLASS_HANDLES_REG = [
Line2D([], [], color=color, marker=CLASS_MARKERS[cls], linewidth=0.9,
label=f"Class {cls}")
for cls, color in enumerate(CLASS_COLORS)
]
WRONG_REG_HANDLE = Line2D([], [], marker="o", markersize=3.5, color="0.2",
linestyle="none", label="|prediction error| > 0.5")
OUTLIER_REG_HANDLE = Line2D([], [], marker="x", markersize=3.5, color="0.2",
linestyle="none", label="appended outlier row")
REG_LEGEND_HANDLES = [*CLASS_HANDLES_REG, WRONG_REG_HANDLE, OUTLIER_REG_HANDLE]
WRONG_HANDLE = Line2D([], [], marker="o", markersize=3.5, color="0.15",
linestyle="none", label="wrong prediction (darker)")
def darken(color, f=0.45):
"""Blend a class color toward black (f = fraction of brightness kept)."""
color = color.lstrip("#")
r, g, b = (int(color[i:i + 2], 16) for i in (0, 2, 4))
return "#{:02x}{:02x}{:02x}".format(round(r * f), round(g * f), round(b * f))
def wrong_clf(df):
"""Rows the frozen classifier assigns to another class than the true one."""
return df["prediction_clf"].to_numpy() != df["cls"].to_numpy()
def wrong_reg(df):
"""Rows whose continuous y prediction misses the observed y by > 0.5."""
return np.abs(df["prediction_reg"].to_numpy() - df["y"].to_numpy()) > 0.5
def overlay(ax, X2, y_true, observed=None, wrong=None, ms=POINT_MS, alpha=1.0):
"""Rows in their true class color (white rims); rows in `wrong` darker."""
if wrong is None:
scatter_classes(ax, X2, y_true, observed=observed, ms=ms,
edgecolor="white", mew=POINT_EDGE, alpha=alpha)
else:
obs = np.ones(len(X2), dtype=bool) if observed is None else observed
scatter_classes(ax, X2, y_true, observed=obs & ~wrong, ms=ms,
edgecolor="white", mew=POINT_EDGE, alpha=alpha)
scatter_classes(ax, X2, y_true, observed=obs & wrong, ms=ms,
edgecolor="white", mew=POINT_EDGE, color_fn=darken, alpha=alpha)
if observed is not None:
outlier_marks(ax, X2, y_true, ~observed, wrong=wrong,
ms=OUTLIER_MS,
markeredgewidth=OUTLIER_EDGE,
show_caption=False)
def boundary_legend(fig, pad_pt=1): # 1.44 pt = 2 px at 100 dpi
"""Two stacked figure legends: classes on line 1, wrong/outliers below.
constrained layout reserves a single outside band per side, so two
legends sharing one "outside" loc overlap. Instead, each legend gets
an explicit bbox: row 1 anchored to the figure top-left, row 2 just
below row 1's measured extent, and the layout rect shrunk so the
axes clear both rows.
"""
outlier_handle = Line2D([], [], marker="x", markersize=3.5, color="0.35",
linestyle="none", label="outliers")
common = dict(fontsize=8, frameon=False, handlelength=1.4,
handletextpad=0.4, columnspacing=1.2)
pad_px = pad_pt * fig.dpi / 72
row1 = fig.legend(handles=[*CLASS_HANDLES], loc="upper left",
bbox_to_anchor=(0, 1), bbox_transform=fig.transFigure,
ncols=4, **common)
fig.canvas.draw()
renderer = fig.canvas.get_renderer()
to_frac = fig.transFigure.inverted()
below_row1 = row1.get_window_extent(renderer).transformed(to_frac).y0
row2 = fig.legend(handles=[WRONG_HANDLE, outlier_handle], loc="upper left",
bbox_to_anchor=(0, 0.95),
bbox_transform=fig.transFigure, ncols=2, **common)
fig.canvas.draw()
occupied = (row1.get_window_extent(renderer).height
+ row2.get_window_extent(renderer).height
+ 3 * pad_px) / fig.bbox.height
fig.get_layout_engine().set(rect=(0, 0, 1, 1 - occupied))
def plot_classifier(ax, predictions, grid, rgba):
"""Draw the classifier map and its case observations on an Axes."""
extent = (grid.xx0.min(), grid.xx0.max(),
grid.xx1.min(), grid.xx1.max())
ax.imshow(rgba, origin="lower", extent=extent, aspect="auto",
interpolation="nearest")
ax.set_xlim(extent[0], extent[1])
ax.set_ylim(extent[2], extent[3])
X2 = predictions[["x", "y"]].to_numpy()
y_true = predictions["cls"].to_numpy()
observed = (None if "outlier" not in predictions
else ~predictions["outlier"].to_numpy())
overlay(ax, X2, y_true, observed, wrong=wrong_clf(predictions))
return ax
def plot_detector(ax, predictions, grid):
"""Draw the detector score map and its case observations on an Axes."""
ax.contourf(grid.xx0, grid.xx1, -grid.response, cmap="PRGn",
levels=21, vmin=-0.35, vmax=0.35, extend="both", alpha=0.5)
ax.contour(grid.xx0, grid.xx1, -grid.response, levels=[0.0],
colors="0.15", linewidths=1.0)
ax.set_xlim(grid.xx0.min(), grid.xx0.max())
ax.set_ylim(grid.xx1.min(), grid.xx1.max())
X2 = predictions[["x", "y"]].to_numpy()
y_true = predictions["cls"].to_numpy()
observed = (None if "outlier" not in predictions
else ~predictions["outlier"].to_numpy())
overlay(ax, X2, y_true, observed)
return ax
def plot_regression(ax, predictions, regressor, x_grid):
"""Draw class-conditioned regression curves and observations on an Axes."""
wrong = wrong_reg(predictions)
appended = (predictions["outlier"].to_numpy() if "outlier" in predictions
else np.zeros(len(predictions), dtype=bool))
y_classes = predictions["cls"].to_numpy()
for cls, color in enumerate(CLASS_COLORS):
model_input = np.column_stack([x_grid, np.full(len(x_grid), cls)])
y_grid = regressor.predict(model_input)
ax.step(x_grid, y_grid, where="post", color=color,
linewidth=0.9, linestyle='-', zorder=4)
selected = y_classes == cls
overlay(ax, predictions.loc[selected, ["x", "y"]].to_numpy(),
y_classes[selected], observed=~appended[selected],
wrong=wrong[selected], alpha=0.5)
return ax
Classifier¶
- Compute the decision boundary
- Create the bakcground (rgba)
- Plots
grid_clf = DecisionBoundaryDisplay.from_estimator(
predictors["clf"], X_train[:, [0,1]], grid_resolution=400, eps=1.5,
response_method="predict_proba");
# background
best = np.argmax(grid_clf.response, axis=-1)
w = np.clip(np.max(grid_clf.response, axis=-1), 0.0, 1.0) ** 0.5
rgba = np.ones((*best.shape, 4))
for k, color in enumerate(CLASS_COLORS):
r, g, b = to_rgb(color)
ix = best == k
for ch, v in ((0, r), (1, g), (2, b)):
rgba[ix, ch] = w[ix] * v + (1.0 - w[ix])
rgba[..., 3] = 0.55
plt.close("all")
rows = [("test half", "test")]
fig, ax = plt.subplots(1, 1, figsize=(4.5, 4.5), constrained_layout=True)
for label, case in rows:
plot_classifier(ax, PREDICTIONS[case], grid_clf, rgba)
ax.set_ylabel("y")
ax.set_title("Classifier")
ax.set_xlabel("x")
boundary_legend(fig)
Outlier Detector¶
grid_det = DecisionBoundaryDisplay.from_estimator(
predictors["det"], X_train[:, [0,1]], grid_resolution=400, eps=1.5,
response_method="decision_function")
# from_estimator leaves its scratch figures open; the grids are only
# consumed as data below, so close them to keep the notebook clean
plt.close("all")
rows = [("test half", "test")]
fig, ax = plt.subplots(1, 1, figsize=(4.5, 4.5), constrained_layout=True)
for label, case in rows:
plot_detector(ax, PREDICTIONS[case], grid_det)
ax.set_ylabel("y")
ax.set_title("Outlier detector")
ax.set_xlabel("x")
fig.colorbar(
ScalarMappable(norm=Normalize(vmin=-0.35, vmax=0.35), cmap="PRGn"),
ax=ax, shrink=0.85, aspect=25, pad=0.02, ticks=[-0.12, 0, 0.12],
).set_label("outlier score")
boundary_legend(fig)
Regressor¶
Here we print the prediction for all X,C as "regression line"
x_values = PREDICTIONS["test"]["x"].to_numpy()
x_grid = np.linspace(x_values.min(), x_values.max(), 500)
fig_reg, axes_reg = plt.subplots(
1, 1, figsize=(5.0, 4), sharex=True, sharey=True,
squeeze=False, constrained_layout=True,
)
class_handles = [
Line2D([], [], color=color, marker=CLASS_MARKERS[cls], linewidth=0.9,
label=f"Class {cls}")
for cls, color in enumerate(CLASS_COLORS)
]
wrong_handle = Line2D([], [], marker="o", markersize=7, color="0.2",
linestyle="none", label="|prediction error| > 0.5")
outlier_handle = Line2D([], [], marker="x", markersize=7, color="0.2",
linestyle="none", label="appended outlier row")
ax = axes_reg[0, 0]
plot_regression(ax, PREDICTIONS["test"], predictors["reg"], x_grid)
ax.set_ylabel("y")
ax.set_xlabel("x")
boundary_legend(fig_reg)
rows = ("test", "direct-drift", "direct-noise", "direct-outliers")
# 2x2 grid mirrors the generated-data layout above: the untouched clean test
# half on top-left so the three transformed variants can be compared at a
# glance
fig, axes = plt.subplots(1, 4, figsize=(9, 3.6), sharex=True, sharey=True,
constrained_layout=True)
for ax, case in zip(axes.flat, rows):
plot_classifier(ax, PREDICTIONS[case], grid_clf, rgba)
ax.set_title(CASE_TITLES[case])
axes[0].set_ylabel("$Y$")
axes[0].set_xlabel("$X$")
boundary_legend(fig)
rows = ("test", "causal-drift", "causal-noise", "causal-outliers")
# 2x2 grid mirrors the generated-data layout above: the untouched clean test
# half on top-left so the three transformed variants can be compared at a
# glance
fig, axes = plt.subplots(1, 4, figsize=(9, 3.6), sharex=True, sharey=True,
constrained_layout=True)
for ax, case in zip(axes.flat, rows):
plot_classifier(ax, PREDICTIONS[case], grid_clf, rgba)
ax.set_title(CASE_TITLES[case])
axes[0].set_ylabel("$Y$")
axes[0].set_xlabel("$X$")
boundary_legend(fig)
Outlier Detector¶
rows = ("test", "direct-drift", "direct-noise", "direct-outliers")
# 2x2 grid mirrors the generated-data layout above: the untouched clean test
# half on top-left so the three transformed variants can be compared at a
# glance
fig, axes = plt.subplots(1, 4, figsize=(9, 3.6), sharex=True, sharey=True,
constrained_layout=True)
for ax, case in zip(axes.flat, rows):
plot_detector(ax, PREDICTIONS[case], grid_det)
ax.set_title(CASE_TITLES[case])
ax.set_xlabel("$X$")
axes[0].set_ylabel("$Y$")
fig.colorbar(
ScalarMappable(norm=Normalize(vmin=-0.35, vmax=0.35), cmap="PRGn"),
ax=axes, shrink=0.85, aspect=25, pad=0.02, ticks=[-0.12, 0, 0.12],
).set_label("outlier score")
boundary_legend(fig)
rows = ("test", "causal-drift", "causal-noise", "causal-outliers")
# 2x2 grid mirrors the generated-data layout above: the untouched clean test
# half on top-left so the three transformed variants can be compared at a
# glance
fig, axes = plt.subplots(1, 4, figsize=(9, 3.6), sharex=True, sharey=True,
constrained_layout=True)
for ax, case in zip(axes.flat, rows):
plot_detector(ax, PREDICTIONS[case], grid_det)
ax.set_title(CASE_TITLES[case])
ax.set_xlabel("$X$")
axes[0].set_ylabel("$Y$")
fig.colorbar(
ScalarMappable(norm=Normalize(vmin=-0.35, vmax=0.35), cmap="PRGn"),
ax=axes, shrink=0.85, aspect=25, pad=0.02, ticks=[-0.12, 0, 0.12],
).set_label("outlier score")
boundary_legend(fig)
Regression¶
rows = ("test", "direct-drift", "direct-noise", "direct-outliers")
fig_reg, axes_reg = plt.subplots(
1, 4, figsize=(9.0, 3.4), sharex=True, sharey=True,
constrained_layout=True,
)
for ax, case in zip(axes_reg.flat, rows):
plot_regression(ax, PREDICTIONS[case], predictors["reg"], X_GRID)
ax.set_title(CASE_TITLES[case])
ax.set_xlabel("$X$")
axes_reg[0].set_ylabel("y")
_ = fig_reg.legend(
handles=REG_LEGEND_HANDLES,
loc="outside upper center", ncols=6, frameon=False,
)
rows = ("test", "causal-drift", "causal-noise", "causal-outliers")
fig_reg, axes_reg = plt.subplots(
1, 4, figsize=(9.0, 3.4), sharex=True, sharey=True,
constrained_layout=True,
)
for ax, case in zip(axes_reg.flat, rows):
plot_regression(ax, PREDICTIONS[case], predictors["reg"], X_GRID)
ax.set_title(CASE_TITLES[case])
ax.set_xlabel("$X$")
axes_reg[0].set_ylabel("y")
_ = fig_reg.legend(
handles=REG_LEGEND_HANDLES,
loc="outside upper center", ncols=6, frameon=False,
)
rows = ("direct-drift", "causal-drift", "direct-noise", "causal-noise")
import string
# 2x2 grid mirrors the generated-data layout above: the untouched clean test
# half on top-left so the three transformed variants can be compared at a
# glance
i = 0
fig, axes = plt.subplots(2, 2, figsize=(4.5, 5), sharex=True, sharey=True,
constrained_layout=True)
for ax, case in zip(axes.flat, rows):
plot_classifier(ax, PREDICTIONS[case], grid_clf, rgba)
ax.set_title(f"({string.ascii_lowercase[i]}) "+CASE_TITLES[case])
ax.set_xlabel("X")
i+=1
axes[0][0].set_ylabel("Y")
axes[1][0].set_ylabel("Y")
boundary_legend(fig, pad_pt=4)