Matplotlib plots of the indeterminate-band trade-off for one task's scores.
Stage: outputs/analysis (offline, after training). Input is the DataFrame from
kirad.utils.get_dataframe_with_scores.
Miscellaneous utility functions for generating plots.
plot_indeterminate_analysis(df, task='depression', show=True)
Plot thresholds and Sn=Sp vs indeterminate budget, plus ROC curves per binary cut.
Analyze performance achievable by including an indeterminate output option and plot.
Parameters:
| Name |
Type |
Description |
Default |
df
|
DataFrame
|
|
required
|
task
|
str
|
|
'depression'
|
show
|
bool
|
|
True
|
Source code in kirad/plotting.py
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87 | def plot_indeterminate_analysis(
df: pd.DataFrame, task: str = "depression", show: bool = True
):
"""Plot thresholds and Sn=Sp vs indeterminate budget, plus ROC curves per binary cut.
Analyze performance achievable by including an indeterminate output option and plot.
Parameters
----------
df: Output of `kirad.utils.get_dataframe_with_scores`
task: one of the tasks the model was trained on, for which scores and labels are in `df`
show: Whether to show the plots; set to False for non-interactive workflows.
"""
# Build one IndetSnSpArray per binary cut `task > i`, for i in 0..max label - 1.
indet_sn_sp_arrays = dict()
labels = df[f"quantized_labels_{task}"]
scores = df[f"scores_{task}"]
for i in range(labels.max()):
indet_sn_sp_arrays[i] = IndetSnSpArray.build(
y_true=(labels > i).astype(int), y_score=scores
)
# Fig 0: lower/upper thresholds vs budget; fig 1: Sn=Sp vs budget; one line per cut.
for i, indet_sn_sp_array in indet_sn_sp_arrays.items():
sn_eq_sp = indet_sn_sp_array.sn_eq_sp_graph()
plt.figure(0)
lines = plt.plot(
sn_eq_sp.indet_frac, sn_eq_sp.lower_thresh, label=f"{task}>{i}"
)
plt.plot(sn_eq_sp.indet_frac, sn_eq_sp.upper_thresh, color=lines[0].get_color())
plt.figure(1)
plt.plot(
sn_eq_sp.indet_frac,
sn_eq_sp.min_sn_sp,
label=f"{task}>{i}",
)
# Label and decorate the thresholds figure (fig 0).
plt.figure(0)
plt.grid(True)
plt.legend()
plt.xlabel("Indeterminate budget (fraction)")
plt.ylabel("Thresholds to achieve optimal Sn=Sp given budget")
plt.title("Upper and lower indeterminacy thresholds as a function of budget")
# Label and decorate the Sn=Sp figure (fig 1).
plt.figure(1)
plt.grid(True)
plt.legend()
plt.xlabel("Indeterminate budget (fraction)")
plt.ylabel("Sn = Sp")
plt.title("Sn=Sp as a function of indeterminate budget")
# One ROC figure per cut, with one curve per indeterminate budget.
for i, indet_sn_sp_array in indet_sn_sp_arrays.items():
plt.figure()
for budget in np.linspace(0.0, 0.7, 8): # HARDCODED: ROC budgets 0%..70%, step 10%
roc_curve = indet_sn_sp_array.roc_curve(budget)
plt.plot(roc_curve.sp, roc_curve.sn, label=f"{budget=:.1f}")
plt.xlim(1.0, 0.0)
plt.ylim(0.0, 1.0)
plt.legend()
plt.xlabel("Specificity")
plt.ylabel("Sensitivity")
plt.title(f"ROC curves with indeterminacy budgets for {task}>{i}")
plt.grid(True)
plt.gca().set_aspect("equal")
# Display all figures unless running non-interactively.
if show:
plt.show()
|