A dataclass used to represent the results of the experiment analysis.
Returned by :meth:AnalysisPlan.analyze <cluster_experiments.inference.analysis_plan.AnalysisPlan.analyze>.
Each row corresponds to one hypothesis test (metric × variant × dimension slice).
Only aggregate fields are stored (ATE, p-value, etc.); for the full underlying
model fit (e.g. statsmodels regression table) for a single run, use
:meth:ExperimentAnalysis.get_inference_results
<cluster_experiments.experiment_analysis.ExperimentAnalysis.get_inference_results>
and then :meth:InferenceResults.model_summary
<cluster_experiments.experiment_analysis.InferenceResults.model_summary>.
Attributes
metric_alias : List[str]
The alias of the metric used in the test
control_variant_name : List[str]
The name of the control variant
treatment_variant_name : List[str]
The name of the treatment variant
control_variant_mean : List[float]
The mean value of the control variant
treatment_variant_mean : List[float]
The mean value of the treatment variant
analysis_type : List[str]
The type of analysis performed
ate : List[float]
The average treatment effect
ate_ci_lower : List[float]
The lower bound of the confidence interval for the ATE
ate_ci_upper : List[float]
The upper bound of the confidence interval for the ATE
p_value : List[float]
The p-value of the test
std_error : List[float]
The standard error of the test
dimension_name : List[str]
The name of the dimension
dimension_value : List[str]
The value of the dimension
alpha: List[float]
The significance level of the test
Source code in cluster_experiments/inference/analysis_results.py
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112 | @dataclass
class AnalysisPlanResults:
"""
A dataclass used to represent the results of the experiment analysis.
Returned by :meth:`AnalysisPlan.analyze <cluster_experiments.inference.analysis_plan.AnalysisPlan.analyze>`.
Each row corresponds to one hypothesis test (metric × variant × dimension slice).
Only aggregate fields are stored (ATE, p-value, etc.); for the full underlying
model fit (e.g. statsmodels regression table) for a single run, use
:meth:`ExperimentAnalysis.get_inference_results
<cluster_experiments.experiment_analysis.ExperimentAnalysis.get_inference_results>`
and then :meth:`InferenceResults.model_summary
<cluster_experiments.experiment_analysis.InferenceResults.model_summary>`.
Attributes
----------
metric_alias : List[str]
The alias of the metric used in the test
control_variant_name : List[str]
The name of the control variant
treatment_variant_name : List[str]
The name of the treatment variant
control_variant_mean : List[float]
The mean value of the control variant
treatment_variant_mean : List[float]
The mean value of the treatment variant
analysis_type : List[str]
The type of analysis performed
ate : List[float]
The average treatment effect
ate_ci_lower : List[float]
The lower bound of the confidence interval for the ATE
ate_ci_upper : List[float]
The upper bound of the confidence interval for the ATE
p_value : List[float]
The p-value of the test
std_error : List[float]
The standard error of the test
dimension_name : List[str]
The name of the dimension
dimension_value : List[str]
The value of the dimension
alpha: List[float]
The significance level of the test
"""
metric_alias: List[str] = field(default_factory=lambda: [])
control_variant_name: List[str] = field(default_factory=lambda: [])
treatment_variant_name: List[str] = field(default_factory=lambda: [])
control_variant_mean: List[float] = field(default_factory=lambda: [])
treatment_variant_mean: List[float] = field(default_factory=lambda: [])
analysis_type: List[str] = field(default_factory=lambda: [])
ate: List[float] = field(default_factory=lambda: [])
ate_ci_lower: List[float] = field(default_factory=lambda: [])
ate_ci_upper: List[float] = field(default_factory=lambda: [])
p_value: List[float] = field(default_factory=lambda: [])
std_error: List[float] = field(default_factory=lambda: [])
dimension_name: List[str] = field(default_factory=lambda: [])
dimension_value: List[str] = field(default_factory=lambda: [])
alpha: List[float] = field(default_factory=lambda: [])
def __add__(self, other):
if not isinstance(other, AnalysisPlanResults):
return NotImplemented
return AnalysisPlanResults(
metric_alias=self.metric_alias + other.metric_alias,
control_variant_name=self.control_variant_name + other.control_variant_name,
treatment_variant_name=self.treatment_variant_name
+ other.treatment_variant_name,
control_variant_mean=self.control_variant_mean + other.control_variant_mean,
treatment_variant_mean=self.treatment_variant_mean
+ other.treatment_variant_mean,
analysis_type=self.analysis_type + other.analysis_type,
ate=self.ate + other.ate,
ate_ci_lower=self.ate_ci_lower + other.ate_ci_lower,
ate_ci_upper=self.ate_ci_upper + other.ate_ci_upper,
p_value=self.p_value + other.p_value,
std_error=self.std_error + other.std_error,
dimension_name=self.dimension_name + other.dimension_name,
dimension_value=self.dimension_value + other.dimension_value,
alpha=self.alpha + other.alpha,
)
def to_dataframe(self):
return pd.DataFrame(asdict(self))
def __str__(self) -> str:
n = len(self.ate)
if n == 0:
return "AnalysisPlanResults(0 tests)"
ate_range = f"[{min(self.ate):.4g}, {max(self.ate):.4g}]"
return f"AnalysisPlanResults({n} test(s): metrics={self.metric_alias}, ATE in {ate_range})"
def summary(self) -> str:
"""Return a summary of the analysis plan results."""
df = self.to_dataframe()
if df.empty:
return "Analysis plan results: no tests."
lines = [
"Analysis plan results",
f" Number of tests: {len(df)}",
"",
df.to_string(index=False),
]
return "\n".join(lines)
|
summary()
Return a summary of the analysis plan results.
Source code in cluster_experiments/inference/analysis_results.py
101
102
103
104
105
106
107
108
109
110
111
112 | def summary(self) -> str:
"""Return a summary of the analysis plan results."""
df = self.to_dataframe()
if df.empty:
return "Analysis plan results: no tests."
lines = [
"Analysis plan results",
f" Number of tests: {len(df)}",
"",
df.to_string(index=False),
]
return "\n".join(lines)
|