Module neuralops.evaluation.ab_testing

Classes

class ABTestResult (metric: str,
mean_a: float,
mean_b: float,
difference: float,
p_value: float,
significant: bool,
better_model: str,
confidence_interval: tuple[float, float])

Expand source code

@dataclass
class ABTestResult:
    metric: str
    mean_a: float
    mean_b: float
    difference: float
    p_value: float
    significant: bool
    better_model: str
    confidence_interval: tuple[float, float]

ABTestResult(metric: str, mean_a: float, mean_b: float, difference: float, p_value: float, significant: bool, better_model: str, confidence_interval: tuple[float, float])

Instance variables

var better_model : str

The type of the None singleton.

var confidence_interval : tuple[float, float]

The type of the None singleton.

var difference : float

The type of the None singleton.

var mean_a : float

The type of the None singleton.

var mean_b : float

The type of the None singleton.

var metric : str

The type of the None singleton.

var p_value : float

The type of the None singleton.

var significant : bool

The type of the None singleton.

class ABTester (alpha: float = 0.05)

Expand source code

class ABTester:
    def __init__(self, alpha: float = 0.05) -> None:
        self.alpha = alpha

    def compare(
        self, results_a: dict[str, list[float]], results_b: dict[str, list[float]]
    ) -> list[ABTestResult]:
        """Compare."""
        outcomes = []
        for metric in results_a:
            if metric not in results_b:
                continue
            a = np.array(results_a[metric])
            b = np.array(results_b[metric])
            t_stat, p_val = stats.ttest_ind(a, b, equal_var=False)
            mean_a, mean_b = np.mean(a), np.mean(b)
            diff = mean_b - mean_a
            se = np.sqrt(np.var(a) / len(a) + np.var(b) / len(b))
            ci = (diff - 1.96 * se, diff + 1.96 * se)
            outcomes.append(
                ABTestResult(
                    metric=metric,
                    mean_a=mean_a,
                    mean_b=mean_b,
                    difference=diff,
                    p_value=p_val,
                    significant=p_val < self.alpha,
                    better_model="B" if diff > 0 else "A",
                    confidence_interval=ci,
                )
            )
        return outcomes

Methods

def compare(self, results_a: dict[str, list[float]], results_b: dict[str, list[float]]) ‑> list[ABTestResult]

Expand source code

def compare(
    self, results_a: dict[str, list[float]], results_b: dict[str, list[float]]
) -> list[ABTestResult]:
    """Compare."""
    outcomes = []
    for metric in results_a:
        if metric not in results_b:
            continue
        a = np.array(results_a[metric])
        b = np.array(results_b[metric])
        t_stat, p_val = stats.ttest_ind(a, b, equal_var=False)
        mean_a, mean_b = np.mean(a), np.mean(b)
        diff = mean_b - mean_a
        se = np.sqrt(np.var(a) / len(a) + np.var(b) / len(b))
        ci = (diff - 1.96 * se, diff + 1.96 * se)
        outcomes.append(
            ABTestResult(
                metric=metric,
                mean_a=mean_a,
                mean_b=mean_b,
                difference=diff,
                p_value=p_val,
                significant=p_val < self.alpha,
                better_model="B" if diff > 0 else "A",
                confidence_interval=ci,
            )
        )
    return outcomes

Compare.