Machine Learning: A/Bテストと実験デザイン — 統計に基づくオンライン実験ガイド

from scipy import stats

# Group A: old model, Group B: new model
conversions_a, n_a = 320, 10000  # 3.2%
conversions_b, n_b = 350, 10000  # 3.5%

z_stat, p_value = stats.proportions_ztest(
    [conversions_a, conversions_b], [n_a, n_b]
)
print(f"p-value: {p_value:.4f}")
print(f"Significant: {p_value < 0.05}")

from scipy import stats
import numpy as np

# Two-proportion z-test
def ab_test_proportions(conversions_a, n_a, conversions_b, n_b, alpha=0.05):
    p_a = conversions_a / n_a
    p_b = conversions_b / n_b
    p_pool = (conversions_a + conversions_b) / (n_a + n_b)

    se = np.sqrt(p_pool * (1 - p_pool) * (1/n_a + 1/n_b))
    z_stat = (p_b - p_a) / se
    p_value = 1 - stats.norm.cdf(z_stat)

    ci_low = (p_b - p_a) - 1.96 * se
    ci_high = (p_b - p_a) + 1.96 * se

    return {
        "conversion_a": f"{p_a:.4f}",
        "conversion_b": f"{p_b:.4f}",
        "lift": f"{(p_b - p_a) / p_a * 100:.2f}%",
        "z_stat": f"{z_stat:.3f}",
        "p_value": f"{p_value:.4f}",
        "ci_95": f"({ci_low:.4f}, {ci_high:.4f})",
        "significant": p_value < alpha,
    }

result = ab_test_proportions(320, 10000, 350, 10000)
for k, v in result.items():
    print(f"{k:15s}: {v}")

# Functions defined successfully

from statsmodels.stats.power import zt_ind_solve_power
from statsmodels.stats.proportion import proportion_effectsize
import numpy as np

def calculate_sample_size(p_baseline, mde, alpha=0.05, power=0.8):
    """Calculate required sample size per group.

    Args:
        p_baseline: Baseline conversion rate
        mde: Minimum Detectable Effect (relative lift)
        alpha: Significance level
        power: Statistical power
    """
    p_new = p_baseline * (1 + mde)
    effect_size = proportion_effectsize(p_new, p_baseline)
    n = zt_ind_solve_power(effect_size=effect_size, alpha=alpha, power=power)
    return int(np.ceil(n))

# Alice's SaaS: baseline 3.2%, wants to detect 10% relative lift
baseline = 0.032
for mde in [0.05, 0.10, 0.15, 0.20]:
    n = calculate_sample_size(baseline, mde)
    print(f"MDE={mde*100:.0f}%: Need {n:,} users per group "
          f"(total {n*2:,}, ~{n*2/baseline/1000:.0f}k visitors)")

# Functions defined successfully

import hashlib

def assign_group(user_id, salt="experiment_1", num_groups=2):
    """Consistent hash assignment for A/B testing."""
    hash_input = f"{salt}_{user_id}".encode()
    hash_val = int(hashlib.md5(hash_input).hexdigest(), 16)
    return hash_val % num_groups  # 0=A, 1=B

# Verify equal split
user_ids = range(100000)
groups = [assign_group(uid) for uid in user_ids]
print(f"Group A: {groups.count(0)} ({groups.count(0)/len(groups)*100:.1f}%)")
print(f"Group B: {groups.count(1)} ({groups.count(1)/len(groups)*100:.1f}%)")

# Functions defined successfully

import numpy as np
from scipy import stats

rng = np.random.default_rng(42)

# Simulate 2-week A/B test for recommendation model
n_users = 20000  # Per group

# Group A: Old model (baseline conversion 3.2%)
conversions_a = rng.binomial(1, 0.032, n_users)
revenue_a = conversions_a * rng.exponential(200, n_users)  # Avg 200 USD per conversion

# Group B: New model (true conversion 3.5%, 9.4% lift)
conversions_b = rng.binomial(1, 0.035, n_users)
revenue_b = conversions_b * rng.exponential(200, n_users)

# Analyze conversion rate difference
conv_rate_a = conversions_a.mean()
conv_rate_b = conversions_b.mean()
z_stat, p_value = stats.proportions_ztest(
    [conversions_a.sum(), conversions_b.sum()],
    [n_users, n_users]
)

# Analyze revenue difference
rev_per_user_a = revenue_a.mean()
rev_per_user_b = revenue_b.mean()
t_stat, t_pvalue = stats.ttest_ind(revenue_a, revenue_b)

print("=" * 50)
print("A/B TEST RESULTS: Recommendation Model")
print("=" * 50)
print(f"Conversion: A={conv_rate_a:.3%} vs B={conv_rate_b:.3%}")
print(f"  Lift: {(conv_rate_b - conv_rate_a) / conv_rate_a * 100:.1f}%")
print(f"  p-value: {p_value:.4f} {'✅ Significant' if p_value < 0.05 else '❌ Not significant'}")

print(f"\nRevenue/User: A={rev_per_user_a:.2f} vs B={rev_per_user_b:.2f} USD")
print(f"  Lift: {(rev_per_user_b - rev_per_user_a) / rev_per_user_a * 100:.1f}%")
print(f"  p-value: {t_pvalue:.4f}")

# Business impact
annual_lift = (rev_per_user_b - rev_per_user_a) * 50000 * 12  # 50k users * 12 months
print(f"\nProjected annual revenue lift: {annual_lift:,.0f} USD")

=
A/B TEST RESULTS: Recommendation Model
=

import numpy as np

rng = np.random.default_rng(42)

# 3 model variants with true conversion rates
true_rates = [0.030, 0.035, 0.028]  # Model B is best
n_arms = len(true_rates)
n_rounds = 10000

# Thompson Sampling
successes = np.zeros(n_arms)
trials = np.zeros(n_arms)
total_reward = 0

for t in range(n_rounds):
    # Sample from Beta distribution for each arm
    samples = [rng.beta(successes[i] + 1, trials[i] - successes[i] + 1) for i in range(n_arms)]
    chosen = np.argmax(samples)

    # Simulate user interaction
    reward = rng.binomial(1, true_rates[chosen])
    successes[chosen] += reward
    trials[chosen] += 1
    total_reward += reward

print(f"Thompson Sampling after {n_rounds} rounds:")
for i in range(n_arms):
    print(f"  Model {i}: chosen {int(trials[i]):5d} times, "
          f"observed rate={successes[i]/max(trials[i],1):.4f} (true: {true_rates[i]:.4f})")
print(f"Total conversions: {int(total_reward)}")

# Executed successfully