import pandas as pd
from kagglehub import KaggleDatasetAdapter
import kagglehub

dataset_dir = kagglehub.dataset_download("prevek18/ames-housing-dataset")
print("Files available:", list(__import__("pathlib").Path(dataset_dir).rglob("*")))

df_ames = kagglehub.dataset_load(
    KaggleDatasetAdapter.PANDAS,
    handle="prevek18/ames-housing-dataset",
    path="AmesHousing.csv",  
    pandas_kwargs={"low_memory": False},  
)

Files available: [PosixPath('/Users/jensreil/.cache/kagglehub/datasets/prevek18/ames-housing-dataset/versions/1/AmesHousing.csv')]

pd.set_option('display.max_columns', None)
df_ames.describe()

import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np

# Numeric-only features
num_df = df_ames.select_dtypes(include=[np.number]).copy()

# Ensure SalePrice is included
if "SalePrice" not in num_df.columns:
    num_df["SalePrice"] = pd.to_numeric(df_ames["SalePrice"], errors="coerce")

# Compute correlation matrix
corr = num_df.corr()

# Big heatmap
plt.figure(figsize=(16, 14))
sns.heatmap(corr, cmap="coolwarm", center=0,
            annot=False, fmt=".2f", square=True,
            cbar_kws={"shrink": 0.75})
plt.title("Full correlation matrix (numeric features including SalePrice)")
plt.show()

# Optional: list strongest correlations with SalePrice
saleprice_corr = corr["SalePrice"].sort_values(ascending=False)
print("=== Top correlations with SalePrice ===")
display(saleprice_corr.head(15))
print("=== Lowest correlations with SalePrice ===")
display(saleprice_corr.tail(15))

=== Top correlations with SalePrice ===

SalePrice         1.000000
Overall Qual      0.799262
Gr Liv Area       0.706780
Garage Cars       0.647877
Garage Area       0.640401
Total Bsmt SF     0.632280
1st Flr SF        0.621676
Year Built        0.558426
Full Bath         0.545604
Year Remod/Add    0.532974
Garage Yr Blt     0.526965
Mas Vnr Area      0.508285
TotRms AbvGrd     0.495474
Fireplaces        0.474558
BsmtFin SF 1      0.432914
Name: SalePrice, dtype: float64

=== Lowest correlations with SalePrice ===

Screen Porch       0.112151
Pool Area          0.068403
Mo Sold            0.035259
3Ssn Porch         0.032225
BsmtFin SF 2       0.005891
Misc Val          -0.015691
Yr Sold           -0.030569
Order             -0.031408
Bsmt Half Bath    -0.035835
Low Qual Fin SF   -0.037660
MS SubClass       -0.085092
Overall Cond      -0.101697
Kitchen AbvGr     -0.119814
Enclosed Porch    -0.128787
PID               -0.246521
Name: SalePrice, dtype: float64

import seaborn as sns
import matplotlib.pyplot as plt

# Ensure numeric only
num_df = df_ames.select_dtypes(include=[np.number]).copy()

# Make sure SalePrice is included
if "SalePrice" not in num_df.columns:
    num_df["SalePrice"] = pd.to_numeric(df_ames["SalePrice"], errors="coerce")

# Compute full correlation matrix
corr = num_df.corr()

# Focus on Gr Liv Area, Overall Qual, and SalePrice correlations
focus_features = ["Gr Liv Area", "Overall Qual", "SalePrice"]
corr_focus = corr[focus_features].sort_values(by="SalePrice", ascending=False)

print("=== Correlations with Gr Liv Area, Overall Qual & SalePrice ===")
display(corr_focus.head(15))  # top 15 correlations with SalePrice

# Collect top correlated features with the three focus features
top_feats = set()
for f in focus_features:
    top_feats.update(
        corr[f].abs().sort_values(ascending=False).head(10).index.tolist()
    )
top_feats = list(top_feats)

# Heatmap for focus + top correlated features
plt.figure(figsize=(12, 10))
sns.heatmap(
    corr.loc[top_feats, top_feats], 
    annot=True, fmt=".2f", cmap="coolwarm", center=0
)
plt.title("Correlation matrix: Gr Liv Area, Overall Qual, SalePrice & related features")

# Save directly to PDF
plt.savefig("correlation_matrix.pdf", format="pdf", bbox_inches="tight")

plt.show()

=== Correlations with Gr Liv Area, Overall Qual & SalePrice ===

import numpy as np
import pandas as pd
import random
import itertools
from itertools import permutations
import warnings

from sklearn.exceptions import ConvergenceWarning
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
from sklearn.neural_network import MLPRegressor
from sklearn.metrics import mean_squared_error, r2_score

# Optional libraries (may or may not be installed)
try:
    import lightgbm as lgb
    HAS_LGB = True
except Exception:
    HAS_LGB = False

try:
    import xgboost as xgb
    HAS_XGB = True
except Exception:
    HAS_XGB = False

def all_subsets_indices(m, letters):
    """Return [(label, comb), ...] where label uses letters (e.g., 'AB'), comb are integer indices."""
    out = []
    for k in range(0, m+1):
        for comb in itertools.combinations(range(m), k):
            label = "nothing" if k == 0 else "".join(letters[j] for j in comb)
            out.append((label, comb))
    return sorted(out, key=lambda t: (len(t[1]), t[1]))

def predict_array(model, arr):
    return np.asarray(model.predict(arr)).reshape(-1)

def shap_for_subset_model(model, x_point_S, x0_S):
    """
    Exact per-point SHAP with mean baseline for k in {0,1,2,3}.
    Returns: (phi_S (len k), f_xS, f_x0S)
    """
    x = x_point_S.reshape(1, -1)
    X0 = x0_S.reshape(1, -1)
    f_x0 = predict_array(model, X0)[0]
    f_x  = predict_array(model, x)[0]
    k = x.shape[1]

    if k == 0:
        return np.array([]), f_x0, f_x0

    if k == 1:
        return np.array([f_x - f_x0]), f_x, f_x0

    if k == 2:
        x1x2 = f_x
        x1x0 = predict_array(model, np.array([[x[0,0], x0_S[1]]]))[0]
        x0x2 = predict_array(model, np.array([[x0_S[0], x[0,1]]]))[0]
        x0x0 = f_x0
        phi1 = 0.5 * ((x1x2 - x0x2) + (x1x0 - x0x0))
        phi2 = 0.5 * ((x1x2 - x1x0) + (x0x2 - x0x0))
        return np.array([phi1, phi2]), f_x, f_x0

    # k == 3
    contrib = np.zeros(3, dtype=float)
    for pi in permutations(range(3)):  # 6 perms
        z = X0.copy(); f_prev = f_x0
        for j in pi:
            z[0, j] = x[0, j]
            f_new = predict_array(model, z)[0]
            contrib[j] += (f_new - f_prev)
            f_prev = f_new
    phi = contrib / 6.0
    return phi, f_x, f_x0

def lattice_predictions_for_row(model, comb, letters, x_point_full, x0_S):
    """
    Build f(∅) and f(subset) for all non-empty subsets ⊆ comb.
    Keys look like: f(∅), f(A), f(B), f(AB), ... using global letters.
    """
    out = {}
    f_empty = float(predict_array(model, x0_S.reshape(1, -1))[0])
    out["f(∅)"] = f_empty

    k = len(comb)
    if k == 0:
        return out

    # Compute f for all non-empty sub-combinations inside comb
    for kk in range(1, k+1):
        for sub in itertools.combinations(comb, kk):
            z = x0_S.copy()
            for j_global in sub:
                pos_in_S = list(comb).index(j_global)
                z[pos_in_S] = x_point_full.iloc[j_global]
            yhat_sub = float(predict_array(model, z.reshape(1, -1))[0])
            label_sub = "f(" + "".join(letters[j] for j in sub) + ")"
            out[label_sub] = yhat_sub

    # Ensure required keys exist (dynamic per subset)
    sub_letters = "".join(letters[j] for j in comb)  # e.g., "AB", "AC", "B", ...
    need = {"f(∅)"}
    if k == 1:
        need |= {f"f({sub_letters})"}
    elif k == 2:
        L1, L2 = sub_letters[0], sub_letters[1]
        need |= {f"f({L1})", f"f({L2})", f"f({L1+L2})"}
    elif k == 3:
        # If the active set is size 3, assume globals are A,B,C
        need |= {"f(A)","f(B)","f(C)","f(AB)","f(AC)","f(BC)","f(ABC)"}

    for kkey in need:
        out.setdefault(kkey, None)

    return out

def mu_from_f_upper_2f(f_dict, pair_letters):
    """
    2-feature μ for an arbitrary pair (e.g., 'AB', 'AC', or 'BC').
    """
    L1, L2 = pair_letters[0], pair_letters[1]
    f0    = float(f_dict["f(∅)"])
    fL1   = float(f_dict[f"f({L1})"])
    fL2   = float(f_dict[f"f({L2})"])
    fL1L2 = float(f_dict[f"f({L1+L2})"])

    nuL1, nuL2, nuL1L2 = (fL1 - f0), (fL2 - f0), (fL1L2 - f0)

    muL1   = nuL1
    muL2   = nuL2
    muL1L2 = nuL1L2 - nuL1 - nuL2

    return {f"μ({L1})": muL1, f"μ({L2})": muL2, f"μ({L1+L2})": muL1L2}

def mu_from_f_upper_3f(f_dict):
    """
    Standard 3-feature μ (assuming letters A,B,C).
    """
    f0   = float(f_dict["f(∅)"])
    fA   = float(f_dict["f(A)"])
    fB   = float(f_dict["f(B)"])
    fC   = float(f_dict["f(C)"])
    fAB  = float(f_dict["f(AB)"])
    fAC  = float(f_dict["f(AC)"])
    fBC  = float(f_dict["f(BC)"])
    fABC = float(f_dict["f(ABC)"])

    nuA, nuB, nuC    = (fA - f0), (fB - f0), (fC - f0)
    nuAB, nuAC, nuBC = (fAB - f0), (fAC - f0), (fBC - f0)
    nuABC = (fABC - f0)

    muA = nuABC - nuBC
    muB = nuABC - nuAC
    muC = nuABC - nuAB
    muAB = -nuC + nuBC + nuAC - nuABC
    muAC = -nuB + nuAB + nuBC - nuABC
    muBC = -nuA + nuAB + nuAC - nuABC
    muABC = nuA+nuB+nuC-nuAB-nuAC-nuBC +nuABC


    return {
        "μ(A)": muA, "μ(B)": muB, "μ(C)": muC,
        "μ(AB)": muAB, "μ(AC)": muAC, "μ(BC)": muBC, "μ(ABC)": muABC
    }

def _subset_strings(letters_str):
    """All non-empty subsets of letters_str (e.g., 'AB' → ['A','B','AB'])"""
    outs = []
    L = list(letters_str)
    for k in range(1, len(L)+1):
        for comb in itertools.combinations(L, k):
            outs.append("".join(comb))
    return outs

def _mu_get(mu_dict, subset_str):
    return float(mu_dict.get(f"μ({subset_str})", 0.0))

def _redistribute_weights(mu_dict, T):
    """
    Recursive weights for coalition T (string like 'AB', 'ABC'), summing to 1 over singletons in T.
    """
    if len(T) == 1:
        return {T: 1.0}

    # all non-empty proper subsets U of T
    proper_subsets = []
    for k in range(1, len(T)):
        for comb in itertools.combinations(T, k):
            proper_subsets.append("".join(comb))

    denom = sum(_mu_get(mu_dict, U) for U in proper_subsets)
    weights = {c: 0.0 for c in T}

    if abs(denom) < 1e-12:
        # equal split to singletons
        equal = 1.0 / len(T)
        for c in T:
            weights[c] += equal
        return weights

    # push mass to U proportional to μ(U), then recurse
    for U in proper_subsets:
        frac = _mu_get(mu_dict, U) / denom
        sub_w = _redistribute_weights(mu_dict, U)
        for c, w in sub_w.items():
            weights[c] += frac * w

    return weights

def reshAP_from_mu_recursive(mu_dict, active_letters):
    """
    ReSHAP_i = sum_{T ⊆ active_letters, i∈T} w_i(T) * μ(T).
    Returns dict: {"ReSHAP A": ..., ...} for active letters only.
    """
    if active_letters == "nothing" or len(active_letters) == 0:
        return {}

    coalitions = _subset_strings(active_letters)
    coalition_weights = {T: _redistribute_weights(mu_dict, T) for T in coalitions}

    res = {f"ReSHAP {c}": 0.0 for c in active_letters}
    for T in coalitions:
        mu_T = _mu_get(mu_dict, T)
        if abs(mu_T) < 1e-12:
            continue
        for c, w in coalition_weights[T].items():
            res[f"ReSHAP {c}"] += w * mu_T
    return res

def make_model_factory(settings, random_state):
    """Return a factory function build_models_for_subset(k) honoring SETTINGS & available libs."""
    def build_models_for_subset(n_features):
        models = {}
        if settings["USE_LINEAR"]:
            models["Linear Regression"] = Pipeline([("model", LinearRegression())])
        if settings["USE_MLP"]:
            models["MLP"] = Pipeline([
                ("scaler", StandardScaler()),
                ("model", MLPRegressor(
                    hidden_layer_sizes=settings["MLP_hidden"],
                    activation="relu", solver="adam",
                    max_iter=settings["MLP_max_iter"],
                    random_state=random_state,
                    early_stopping=settings["MLP_early_stopping"],
                    n_iter_no_change=settings["MLP_n_iter_no_change"],
                    tol=settings["MLP_tol"],
                    learning_rate_init=settings["MLP_lr"],
                    alpha=settings["MLP_alpha"]
                ))
            ])
        if settings["USE_LGB"] and HAS_LGB:
            models["LightGBM"] = lgb.LGBMRegressor(
                n_estimators=500, learning_rate=0.05, num_leaves=31,
                random_state=random_state, verbose=-1
            )
        if settings["USE_XGB"] and HAS_XGB:
            models["XGBoost"] = xgb.XGBRegressor(
                n_estimators=500, learning_rate=0.05, max_depth=6,
                subsample=0.8, colsample_bytree=0.8,
                objective="reg:squarederror",
                random_state=random_state, n_jobs=-1
            )
        return models
    return build_models_for_subset

def run_pipeline(settings):
    """
    End-to-end:
    - slice data by MODE
    - split train/test
    - pick one or many test rows (per settings)
    - train each subset model per chosen algorithms (once)
    - for each chosen test row: compute SHAP, lattice f, μ, and ReSHAP
    - display per-model tables per test row
    """
    # Warnings
    if settings["SILENCE_MLP_WARNINGS"]:
        warnings.filterwarnings("ignore", category=ConvergenceWarning)
    else:
        warnings.filterwarnings("default", category=ConvergenceWarning)

    # ---- Data slice
    MODE = settings["MODE"]
    RANDOM_STATE = settings["RANDOM_STATE"]

    df = df_ames.copy()
    df.columns = df.columns.str.strip()

    need = ["Gr Liv Area", "Overall Qual", "SalePrice"]
    if MODE == "totrms":
        need += ["TotRms AbvGrd"]
    base_df = df[need].apply(pd.to_numeric, errors="coerce").dropna()

    if MODE == "two":
        X_full = base_df[["Gr Liv Area", "Overall Qual"]].copy()
        feature_names_full = ["Gr Liv Area", "Overall Qual"]
    elif MODE == "dup_grliv":
        X_full = base_df[["Gr Liv Area", "Overall Qual"]].copy()
        X_full["Gr Liv Area (DUP)"] = X_full["Gr Liv Area"]
        feature_names_full = ["Gr Liv Area", "Overall Qual", "Gr Liv Area (DUP)"]
    elif MODE == "dup_qual":
        X_full = base_df[["Gr Liv Area", "Overall Qual"]].copy()
        X_full["Overall Qual (DUP)"] = X_full["Overall Qual"]
        feature_names_full = ["Gr Liv Area", "Overall Qual", "Overall Qual (DUP)"]
    elif MODE == "totrms":
        X_full = base_df[["Gr Liv Area", "Overall Qual", "TotRms AbvGrd"]].copy()
        feature_names_full = ["Gr Liv Area", "Overall Qual", "TotRms AbvGrd"]
    else:
        raise ValueError("Invalid MODE")

    y_full = base_df["SalePrice"]
    m = len(feature_names_full)
    letters = "ABC"[:m]

    X_tr_full, X_te_full, y_tr, y_te = train_test_split(
        X_full, y_full, test_size=0.2, random_state=RANDOM_STATE
    )

    # ---- Pick one or many test rows
    if settings["PICK_RANDOM_POINT"]:
        random.seed(RANDOM_STATE)
        n = min(int(settings.get("N_RANDOM_POINTS", 1)), len(X_te_full))
        test_indices = random.sample(list(X_te_full.index), n)
    else:
        if settings["CUSTOM_TEST_INDEX"] is not None:
            idx = settings["CUSTOM_TEST_INDEX"]
            if idx not in X_te_full.index:
                raise ValueError(f"CUSTOM_TEST_INDEX {idx} not in test indices.")
            test_indices = [idx]
        else:
            test_indices = [X_te_full.index[0]]

    print(f"Mode: {MODE} | Features (A,B{',C' if m==3 else ''}): {list(zip(list(letters), feature_names_full))}")
    print(f"Evaluating {len(test_indices)} test point(s): {test_indices}")

    # ---- Build subsets & models
    build_models_for_subset = make_model_factory(settings, RANDOM_STATE)
    subsets = all_subsets_indices(m, letters)

    # Determine model order (respect settings + availability)
    model_names = list(build_models_for_subset(1).keys())
    if settings["USE_LGB"] and HAS_LGB and "LightGBM" not in model_names:
        model_names.append("LightGBM")
    if settings["USE_XGB"] and HAS_XGB and "XGBoost" not in model_names:
        model_names.append("XGBoost")

    # ---- Train per subset (once)
    fitted = {}  # (model_name, label) -> dict(model, r2, rmse, x0_S)
    for label, comb in subsets:
        k = len(comb)
        if k == 0:
            # Constant model
            y_hat_test = np.full_like(y_te.values, fill_value=y_tr.mean(), dtype=float)
            r2  = r2_score(y_te.values, y_hat_test)
            rmse = float(np.sqrt(mean_squared_error(y_te.values, y_hat_test)))
            for mn in model_names:
                fitted[(mn, label)] = {"model": None, "r2": r2, "rmse": rmse, "x0_S": None}
            continue

        X_tr_S = X_tr_full.iloc[:, list(comb)].values
        X_te_S = X_te_full.iloc[:, list(comb)].values

        # Baseline
        bl = settings["BASELINE"]
        if bl == "mean":
            x0_S = X_tr_full.iloc[:, list(comb)].mean(axis=0).values.astype(float)
        elif bl == "median":
            x0_S = X_tr_full.iloc[:, list(comb)].median(axis=0).values.astype(float)
        elif bl == "zero":
            x0_S = np.zeros(len(comb), dtype=float)
        else:
            raise ValueError("BASELINE must be one of {'mean','median','zero'}")

        for mn, mdl in build_models_for_subset(k).items():
            mdl.fit(X_tr_S, y_tr.values)
            y_hat_test = mdl.predict(X_te_S)
            r2  = r2_score(y_te.values, y_hat_test)
            rmse = float(np.sqrt(mean_squared_error(y_te.values, y_hat_test)))
            fitted[(mn, label)] = {"model": mdl, "r2": r2, "rmse": rmse, "x0_S": x0_S}

    # ---- Evaluate each chosen point
    for idx in test_indices:
        x_point_full = X_te_full.loc[idx]
        y_true = y_te.loc[idx]

        print(f"\n──────── Test point index: {idx} | True SalePrice: {y_true} ────────")
        display(pd.DataFrame(x_point_full).T)

        for mn in model_names:
            rows = []
            for label, comb in subsets:
                k = len(comb)
                entry = fitted[(mn, label)]
                r2, rmse = entry["r2"], entry["rmse"]

                # defaults
                phi_cols = [""]*m
                mu_cols  = {}
                resh_cols = {}
                lattice_dict = {}
                baseline = float(y_tr.mean()) if k==0 else None
                prediction = baseline

                if k == 0:
                    lattice_dict = {"f(∅)": baseline}
                else:
                    model = entry["model"]
                    x0_S = entry["x0_S"]
                    x_point_S = x_point_full.iloc[list(comb)].values

                    # φ for this point
                    phi_S, f_xS, f_x0S = shap_for_subset_model(model, x_point_S, x0_S)
                    baseline  = float(f_x0S)
                    prediction = float(f_xS)

                    # Align φ into global A,B,(C)
                    for j in range(m):
                        if j in comb:
                            pos = list(comb).index(j)
                            phi_cols[j] = float(phi_S[pos])
                        else:
                            phi_cols[j] = ""

                    # Lattice
                    lattice_dict = lattice_predictions_for_row(model, comb, letters, x_point_full, x0_S)

                    # μ and ReSHAP
                    if k == 2:
                        sub_letters = "".join(letters[j] for j in comb)  # "AB"/"AC"/"BC"
                        mu_cols = mu_from_f_upper_2f(lattice_dict, sub_letters)
                        resh_cols = reshAP_from_mu_recursive(mu_cols, sub_letters)
                    elif k == 3:
                        mu_cols = mu_from_f_upper_3f(lattice_dict)
                        resh_cols = reshAP_from_mu_recursive(mu_cols, "ABC")
                    elif k == 1:
                        sl = "".join(letters[j] for j in comb)  # "A"/"B"/"C"
                        f0 = float(lattice_dict["f(∅)"])
                        fA = float(lattice_dict[f"f({sl})"])
                        mu_cols = {f"μ({sl})": (fA - f0)}
                        resh_cols = reshAP_from_mu_recursive(mu_cols, sl)

                # row
                row = {
                    "Feature subset": label,
                    "n_features": k,
                    "R2 (subset test)": r2,
                    "RMSE (subset test)": rmse,
                    "Baseline f(∅)": baseline,
                    "Prediction f(S)": prediction,
                    "φ A": phi_cols[0] if m>=1 else "",
                    "φ B": phi_cols[1] if m>=2 else "",
                    "φ C": phi_cols[2] if m>=3 else "",
                    "f(∅)": lattice_dict.get("f(∅)", ""),
                    "f(A)": lattice_dict.get("f(A)", ""),
                    "f(B)": lattice_dict.get("f(B)", ""),
                    "f(AB)": lattice_dict.get("f(AB)", ""),
                }
                if m == 3:
                    row.update({
                        "f(C)": lattice_dict.get("f(C)", ""),
                        "f(AC)": lattice_dict.get("f(AC)", ""),
                        "f(BC)": lattice_dict.get("f(BC)", ""),
                        "f(ABC)": lattice_dict.get("f(ABC)", ""),
                    })
                for key in ["μ(A)","μ(B)","μ(C)","μ(AB)","μ(AC)","μ(BC)","μ(ABC)"]:
                    row[key] = mu_cols.get(key, "")
                for key in ["ReSHAP A","ReSHAP B","ReSHAP C"]:
                    row[key] = resh_cols.get(key, "")

                rows.append(row)

            # display table for this point + model
            base_cols = ["Feature subset","n_features","R2 (subset test)","RMSE (subset test)",
                         "Baseline f(∅)","Prediction f(S)","φ A","φ B"]
            if m==3: base_cols += ["φ C"]
            lattice_cols = ["f(∅)","f(A)","f(B)","f(AB)"]
            if m==3: lattice_cols += ["f(C)","f(AC)","f(BC)","f(ABC)"]
            mu_cols_all = ["μ(A)","μ(B)","μ(AB)"]
            if m==3: mu_cols_all = ["μ(A)","μ(B)","μ(C)","μ(AB)","μ(AC)","μ(BC)","μ(ABC)"]
            resh_cols_all = ["ReSHAP A","ReSHAP B"]
            if m==3: resh_cols_all += ["ReSHAP C"]

            ordered_cols = base_cols + lattice_cols + mu_cols_all + resh_cols_all
            tbl = pd.DataFrame(rows)[ordered_cols]
            print(f"\n=== Subset predictions + SHAP + μ + ReSHAP — {mn} | Test idx {idx} ===")
            display(tbl)

SETTINGS = dict(
    # Data slice / feature modes:
    MODE="two",           # 'two' | 'dup_grliv' | 'dup_qual' | 'totrms'

    # Baseline used inside each subset for f(∅):
    BASELINE="mean",           # 'mean' | 'median' | 'zero'

    # Randomness & test row:
    RANDOM_STATE=42,
    PICK_RANDOM_POINT=True,    # True → draw a random test row (with the seed)
    N_RANDOM_POINTS=10,         # ← NEW: number of distinct random test points to assess
    CUSTOM_TEST_INDEX=None,    # if PICK_RANDOM_POINT=False, use this test index (must be in X_te)

    # Which models to run:
    USE_LINEAR=True,
    USE_MLP=True,
    USE_LGB=True,              # will only run if LightGBM is installed
    USE_XGB=True,              # will only run if XGBoost is installed

    # MLP hyperparameters (used if USE_MLP=True):
    MLP_hidden=(64, 32),
    MLP_max_iter=5000,
    MLP_early_stopping=True,
    MLP_n_iter_no_change=20,
    MLP_tol=1e-4,
    MLP_lr=1e-3,
    MLP_alpha=1e-4,

    # Warnings:
    SILENCE_MLP_WARNINGS=True
)

# ---- Go! (expects df_ames to already be in memory) ----
run_pipeline(SETTINGS)

Mode: two | Features (A,B): [('A', 'Gr Liv Area'), ('B', 'Overall Qual')]
Evaluating 10 test point(s): [1093, 1032, 2783, 411, 1385, 1577, 1427, 518, 2181, 1880]

──────── Test point index: 1093 | True SalePrice: 226750 ────────

=== Subset predictions + SHAP + μ + ReSHAP — Linear Regression | Test idx 1093 ===

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 1093 ===

=== Subset predictions + SHAP + μ + ReSHAP — LightGBM | Test idx 1093 ===

=== Subset predictions + SHAP + μ + ReSHAP — XGBoost | Test idx 1093 ===

──────── Test point index: 1032 | True SalePrice: 306000 ────────

=== Subset predictions + SHAP + μ + ReSHAP — Linear Regression | Test idx 1032 ===

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 1032 ===

=== Subset predictions + SHAP + μ + ReSHAP — LightGBM | Test idx 1032 ===

=== Subset predictions + SHAP + μ + ReSHAP — XGBoost | Test idx 1032 ===

SETTINGS = dict(
    # Data slice / feature modes:
    MODE="dup_qual",           # 'two' | 'dup_grliv' | 'dup_qual' | 'totrms'

    # Baseline used inside each subset for f(∅):
    BASELINE="mean",           # 'mean' | 'median' | 'zero'

    # Randomness & test row:
    RANDOM_STATE=42,
    PICK_RANDOM_POINT=True,    # True → draw a random test row (with the seed)
    N_RANDOM_POINTS=10,         # ← NEW: number of distinct random test points to assess
    CUSTOM_TEST_INDEX=None,    # if PICK_RANDOM_POINT=False, use this test index (must be in X_te)

    # Which models to run:
    USE_LINEAR=True,
    USE_MLP=True,
    USE_LGB=True,              # will only run if LightGBM is installed
    USE_XGB=True,              # will only run if XGBoost is installed

    # MLP hyperparameters (used if USE_MLP=True):
    MLP_hidden=(64, 32),
    MLP_max_iter=5000,
    MLP_early_stopping=True,
    MLP_n_iter_no_change=20,
    MLP_tol=1e-4,
    MLP_lr=1e-3,
    MLP_alpha=1e-4,

    # Warnings:
    SILENCE_MLP_WARNINGS=True
)

# ---- Go! (expects df_ames to already be in memory) ----
run_pipeline(SETTINGS)

Mode: dup_qual | Features (A,B,C): [('A', 'Gr Liv Area'), ('B', 'Overall Qual'), ('C', 'Overall Qual (DUP)')]
Evaluating 10 test point(s): [1093, 1032, 2783, 411, 1385, 1577, 1427, 518, 2181, 1880]

──────── Test point index: 1093 | True SalePrice: 226750 ────────

=== Subset predictions + SHAP + μ + ReSHAP — Linear Regression | Test idx 1093 ===

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 1093 ===

=== Subset predictions + SHAP + μ + ReSHAP — LightGBM | Test idx 1093 ===

=== Subset predictions + SHAP + μ + ReSHAP — XGBoost | Test idx 1093 ===

──────── Test point index: 1032 | True SalePrice: 306000 ────────

=== Subset predictions + SHAP + μ + ReSHAP — Linear Regression | Test idx 1032 ===

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 1032 ===

=== Subset predictions + SHAP + μ + ReSHAP — LightGBM | Test idx 1032 ===

=== Subset predictions + SHAP + μ + ReSHAP — XGBoost | Test idx 1032 ===

# ============================================================
# 6) Settings & Run (edit here, then run this cell)
# ============================================================

SETTINGS = dict(
    # Data slice / feature modes:
    MODE="dup_grliv",           # 'two' | 'dup_grliv' | 'dup_qual' | 'totrms'

    # Baseline used inside each subset for f(∅):
    BASELINE="mean",           # 'mean' | 'median' | 'zero'

    # Randomness & test row:
    RANDOM_STATE=42,
    PICK_RANDOM_POINT=True,    # True → draw a random test row (with the seed)
    N_RANDOM_POINTS=10,         # ← NEW: number of distinct random test points to assess
    CUSTOM_TEST_INDEX=None,    # if PICK_RANDOM_POINT=False, use this test index (must be in X_te)

    # Which models to run:
    USE_LINEAR=True,
    USE_MLP=True,
    USE_LGB=True,              # will only run if LightGBM is installed
    USE_XGB=True,              # will only run if XGBoost is installed

    # MLP hyperparameters (used if USE_MLP=True):
    MLP_hidden=(64, 32),
    MLP_max_iter=5000,
    MLP_early_stopping=True,
    MLP_n_iter_no_change=20,
    MLP_tol=1e-4,
    MLP_lr=1e-3,
    MLP_alpha=1e-4,

    # Warnings:
    SILENCE_MLP_WARNINGS=True
)

# ---- Go! (expects df_ames to already be in memory) ----
run_pipeline(SETTINGS)

Mode: dup_grliv | Features (A,B,C): [('A', 'Gr Liv Area'), ('B', 'Overall Qual'), ('C', 'Gr Liv Area (DUP)')]
Evaluating 10 test point(s): [1093, 1032, 2783, 411, 1385, 1577, 1427, 518, 2181, 1880]

──────── Test point index: 1093 | True SalePrice: 226750 ────────

=== Subset predictions + SHAP + μ + ReSHAP — Linear Regression | Test idx 1093 ===

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 1093 ===

=== Subset predictions + SHAP + μ + ReSHAP — LightGBM | Test idx 1093 ===

=== Subset predictions + SHAP + μ + ReSHAP — XGBoost | Test idx 1093 ===

──────── Test point index: 1032 | True SalePrice: 306000 ────────

=== Subset predictions + SHAP + μ + ReSHAP — Linear Regression | Test idx 1032 ===

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 1032 ===

=== Subset predictions + SHAP + μ + ReSHAP — LightGBM | Test idx 1032 ===

=== Subset predictions + SHAP + μ + ReSHAP — XGBoost | Test idx 1032 ===

# ============================================================
# 6) Settings & Run (edit here, then run this cell)
# ============================================================

SETTINGS = dict(
    # Data slice / feature modes:
    MODE="totrms",           # 'two' | 'dup_grliv' | 'dup_qual' | 'totrms'

    # Baseline used inside each subset for f(∅):
    BASELINE="mean",           # 'mean' | 'median' | 'zero'

    # Randomness & test row:
    RANDOM_STATE=42,
    PICK_RANDOM_POINT=True,    # True → draw a random test row (with the seed)
    N_RANDOM_POINTS=10,         # ← NEW: number of distinct random test points to assess
    CUSTOM_TEST_INDEX=None,    # if PICK_RANDOM_POINT=False, use this test index (must be in X_te)

    # Which models to run:
    USE_LINEAR=True,
    USE_MLP=True,
    USE_LGB=True,              # will only run if LightGBM is installed
    USE_XGB=True,              # will only run if XGBoost is installed

    # MLP hyperparameters (used if USE_MLP=True):
    MLP_hidden=(64, 32),
    MLP_max_iter=5000,
    MLP_early_stopping=True,
    MLP_n_iter_no_change=20,
    MLP_tol=1e-4,
    MLP_lr=1e-3,
    MLP_alpha=1e-4,

    # Warnings:
    SILENCE_MLP_WARNINGS=True
)

# ---- Go! (expects df_ames to already be in memory) ----
run_pipeline(SETTINGS)

Mode: totrms | Features (A,B,C): [('A', 'Gr Liv Area'), ('B', 'Overall Qual'), ('C', 'TotRms AbvGrd')]
Evaluating 10 test point(s): [1093, 1032, 2783, 411, 1385, 1577, 1427, 518, 2181, 1880]

──────── Test point index: 1093 | True SalePrice: 226750 ────────

=== Subset predictions + SHAP + μ + ReSHAP — Linear Regression | Test idx 1093 ===

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 1093 ===

=== Subset predictions + SHAP + μ + ReSHAP — LightGBM | Test idx 1093 ===

=== Subset predictions + SHAP + μ + ReSHAP — XGBoost | Test idx 1093 ===

──────── Test point index: 1032 | True SalePrice: 306000 ────────

=== Subset predictions + SHAP + μ + ReSHAP — Linear Regression | Test idx 1032 ===

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 1032 ===

=== Subset predictions + SHAP + μ + ReSHAP — LightGBM | Test idx 1032 ===

=== Subset predictions + SHAP + μ + ReSHAP — XGBoost | Test idx 1032 ===

# ============================================================
# 5) Model factory (parameterized) + training/eval runner
# ============================================================
def make_model_factory(settings, random_state):
    """Return a factory function build_models_for_subset(k) honoring SETTINGS & available libs."""
    def build_models_for_subset(n_features):
        models = {}
        if settings["USE_LINEAR"]:
            models["Linear Regression"] = Pipeline([("model", LinearRegression())])
        if settings["USE_MLP"]:
            models["MLP"] = Pipeline([
                ("scaler", StandardScaler()),
                ("model", MLPRegressor(
                    hidden_layer_sizes=settings["MLP_hidden"],
                    activation="relu", solver="adam",
                    max_iter=settings["MLP_max_iter"],
                    random_state=random_state,
                    early_stopping=settings["MLP_early_stopping"],
                    n_iter_no_change=settings["MLP_n_iter_no_change"],
                    tol=settings["MLP_tol"],
                    learning_rate_init=settings["MLP_lr"],
                    alpha=settings["MLP_alpha"]
                ))
            ])
        if settings["USE_LGB"] and HAS_LGB:
            models["LightGBM"] = lgb.LGBMRegressor(
                n_estimators=500, learning_rate=0.05, num_leaves=31,
                random_state=random_state, verbose=-1
            )
        if settings["USE_XGB"] and HAS_XGB:
            models["XGBoost"] = xgb.XGBRegressor(
                n_estimators=500, learning_rate=0.05, max_depth=6,
                subsample=0.8, colsample_bytree=0.8,
                objective="reg:squarederror",
                random_state=random_state, n_jobs=-1
            )
        return models
    return build_models_for_subset

def _num(x):
    try:
        return float(x)
    except Exception:
        return np.nan

def compute_P_R_metrics(tbl):
    """
    Compute diagnostics + P, R, P/R from AB and ABC rows.
    Returns dict with raw f-values, ψ, ReSHAP, denominators, and metrics.
    """
    out = {"P": np.nan, "R": np.nan, "P/R": np.nan}

    try:
        row_ab  = tbl.loc[tbl["Feature subset"] == "AB"].iloc[0]
        row_abc = tbl.loc[tbl["Feature subset"] == "ABC"].iloc[0]
    except Exception:
        return out

    # --- raw lattice values ---
    f_ab_s  = _num(row_ab["f(AB)"])
    f_ab_a  = _num(row_ab["f(∅)"])
    f_abc_s = _num(row_abc["f(ABC)"])
    f_abc_a = _num(row_abc["f(∅)"])

    # --- SHAP ψ values ---
    psi_ab_a   = abs(_num(row_ab["φ B"]))
    psi_abc_a  = abs(_num(row_abc["φ B"]))


    # --- ReSHAP values ---
    resh_ab_a  = abs(_num(row_ab["ReSHAP B"]))
    resh_abc_a = abs(_num(row_abc["ReSHAP B"]))

    # --- denominators ---
    d_ab  = abs(f_ab_s  - f_ab_a)
    d_abc = abs(f_abc_s - f_abc_a)

    # safe fractions
    with np.errstate(divide="ignore", invalid="ignore"):
        p_ab  = psi_ab_a  / d_ab  if d_ab  not in (0, np.nan) else np.nan
        p_abc = psi_abc_a / d_abc if d_abc not in (0, np.nan) else np.nan
        r_ab  = resh_ab_a / d_ab  if d_ab  not in (0, np.nan) else np.nan
        r_abc = resh_abc_a/ d_abc if d_abc not in (0, np.nan) else np.nan

        P = p_ab - p_abc if p_abc not in (0, np.nan) else np.nan
        R = r_ab - r_abc if r_abc not in (0, np.nan) else np.nan

        PR = P / R       if R    not in (0, np.nan) else np.nan

    return {
        # raw values
        "f_AB(S)": f_ab_s, "f_AB(A)": f_ab_a,
        "f_ABC(S)": f_abc_s, "f_ABC(A)": f_abc_a,
        # diagnostics
        "psi_AB(A)": psi_ab_a, "den_AB": d_ab,
        "psi_ABC(A)": psi_abc_a, "den_ABC": d_abc,
        "ReSHAP_AB(A)": resh_ab_a, "ReSHAP_ABC(A)": resh_abc_a,
        # metrics
        "P": P, "R": R, "P/R": PR
    }


def run_pipeline(settings):
    """
    End-to-end:
    - slice data by MODE
    - split train/test
    - pick one or many test rows (per settings)
    - train each subset model per chosen algorithms (once)
    - for each chosen test row: compute SHAP, lattice f, μ, and ReSHAP
    - display per-model tables per test row
    """
    # Warnings
    if settings["SILENCE_MLP_WARNINGS"]:
        warnings.filterwarnings("ignore", category=ConvergenceWarning)
    else:
        warnings.filterwarnings("default", category=ConvergenceWarning)

    # ---- Data slice
    MODE = settings["MODE"]
    RANDOM_STATE = settings["RANDOM_STATE"]

    df = df_ames.copy()
    df.columns = df.columns.str.strip()

    need = ["Gr Liv Area", "Overall Qual", "SalePrice"]
    if MODE == "totrms":
        need += ["TotRms AbvGrd"]
    base_df = df[need].apply(pd.to_numeric, errors="coerce").dropna()

    if MODE == "two":
        X_full = base_df[["Gr Liv Area", "Overall Qual"]].copy()
        feature_names_full = ["Gr Liv Area", "Overall Qual"]
    elif MODE == "dup_grliv":
        X_full = base_df[["Gr Liv Area", "Overall Qual"]].copy()
        X_full["Gr Liv Area (DUP)"] = X_full["Gr Liv Area"]
        feature_names_full = ["Gr Liv Area", "Overall Qual", "Gr Liv Area (DUP)"]
    elif MODE == "dup_qual":
        X_full = base_df[["Gr Liv Area", "Overall Qual"]].copy()
        X_full["Overall Qual (DUP)"] = X_full["Overall Qual"]
        feature_names_full = ["Gr Liv Area", "Overall Qual", "Overall Qual (DUP)"]
    elif MODE == "totrms":
        X_full = base_df[["Gr Liv Area", "Overall Qual", "TotRms AbvGrd"]].copy()
        feature_names_full = ["Gr Liv Area", "Overall Qual", "TotRms AbvGrd"]
    else:
        raise ValueError("Invalid MODE")

    y_full = base_df["SalePrice"]
    m = len(feature_names_full)
    letters = "ABC"[:m]

    X_tr_full, X_te_full, y_tr, y_te = train_test_split(
        X_full, y_full, test_size=0.2, random_state=RANDOM_STATE
    )

    # ---- Pick one or many test rows
    if settings["PICK_RANDOM_POINT"]:
        random.seed(RANDOM_STATE)
        n = min(int(settings.get("N_RANDOM_POINTS", 1)), len(X_te_full))
        test_indices = random.sample(list(X_te_full.index), n)
    else:
        if settings["CUSTOM_TEST_INDEX"] is not None:
            idx = settings["CUSTOM_TEST_INDEX"]
            if idx not in X_te_full.index:
                raise ValueError(f"CUSTOM_TEST_INDEX {idx} not in test indices.")
            test_indices = [idx]
        else:
            test_indices = [X_te_full.index[0]]

    print(f"Mode: {MODE} | Features (A,B{',C' if m==3 else ''}): {list(zip(list(letters), feature_names_full))}")
    print(f"Evaluating {len(test_indices)} test point(s): {test_indices}")

    # ---- Build subsets & models
    build_models_for_subset = make_model_factory(settings, RANDOM_STATE)
    subsets = all_subsets_indices(m, letters)

    # Determine model order (respect settings + availability)
    model_names = list(build_models_for_subset(1).keys())
    if settings["USE_LGB"] and HAS_LGB and "LightGBM" not in model_names:
        model_names.append("LightGBM")
    if settings["USE_XGB"] and HAS_XGB and "XGBoost" not in model_names:
        model_names.append("XGBoost")

    # container: model_name -> list of dicts {"P":..,"R":..,"P/R":..}
    model_metrics = {mn: [] for mn in model_names}
    
    # ---- Train per subset (once)
    fitted = {}  # (model_name, label) -> dict(model, r2, rmse, x0_S)
    for label, comb in subsets:
        k = len(comb)
        if k == 0:
            # Constant model
            y_hat_test = np.full_like(y_te.values, fill_value=y_tr.mean(), dtype=float)
            r2  = r2_score(y_te.values, y_hat_test)
            rmse = float(np.sqrt(mean_squared_error(y_te.values, y_hat_test)))
            for mn in model_names:
                fitted[(mn, label)] = {"model": None, "r2": r2, "rmse": rmse, "x0_S": None}
            continue

        X_tr_S = X_tr_full.iloc[:, list(comb)].values
        X_te_S = X_te_full.iloc[:, list(comb)].values

        # Baseline
        bl = settings["BASELINE"]
        if bl == "mean":
            x0_S = X_tr_full.iloc[:, list(comb)].mean(axis=0).values.astype(float)
        elif bl == "median":
            x0_S = X_tr_full.iloc[:, list(comb)].median(axis=0).values.astype(float)
        elif bl == "zero":
            x0_S = np.zeros(len(comb), dtype=float)
        else:
            raise ValueError("BASELINE must be one of {'mean','median','zero'}")

        for mn, mdl in build_models_for_subset(k).items():
            mdl.fit(X_tr_S, y_tr.values)
            y_hat_test = mdl.predict(X_te_S)
            r2  = r2_score(y_te.values, y_hat_test)
            rmse = float(np.sqrt(mean_squared_error(y_te.values, y_hat_test)))
            fitted[(mn, label)] = {"model": mdl, "r2": r2, "rmse": rmse, "x0_S": x0_S}

    # ---- Evaluate each chosen point
    for idx in test_indices:
        x_point_full = X_te_full.loc[idx]
        y_true = y_te.loc[idx]

        print(f"\n──────── Test point index: {idx} | True SalePrice: {y_true} ────────")
        display(pd.DataFrame(x_point_full).T)

        for mn in model_names:
            rows = []
            for label, comb in subsets:
                k = len(comb)
                entry = fitted[(mn, label)]
                r2, rmse = entry["r2"], entry["rmse"]

                # defaults
                phi_cols = [""]*m
                mu_cols  = {}
                resh_cols = {}
                lattice_dict = {}
                baseline = float(y_tr.mean()) if k==0 else None
                prediction = baseline

                if k == 0:
                    lattice_dict = {"f(∅)": baseline}
                else:
                    model = entry["model"]
                    x0_S = entry["x0_S"]
                    x_point_S = x_point_full.iloc[list(comb)].values

                    # φ for this point
                    phi_S, f_xS, f_x0S = shap_for_subset_model(model, x_point_S, x0_S)
                    baseline  = float(f_x0S)
                    prediction = float(f_xS)

                    # Align φ into global A,B,(C)
                    for j in range(m):
                        if j in comb:
                            pos = list(comb).index(j)
                            phi_cols[j] = float(phi_S[pos])
                        else:
                            phi_cols[j] = ""

                    # Lattice
                    lattice_dict = lattice_predictions_for_row(model, comb, letters, x_point_full, x0_S)

                    # μ and ReSHAP
                    if k == 2:
                        sub_letters = "".join(letters[j] for j in comb)  # "AB"/"AC"/"BC"
                        mu_cols = mu_from_f_upper_2f(lattice_dict, sub_letters)
                        resh_cols = reshAP_from_mu_recursive(mu_cols, sub_letters)
                    elif k == 3:
                        mu_cols = mu_from_f_upper_3f(lattice_dict)
                        resh_cols = reshAP_from_mu_recursive(mu_cols, "ABC")
                    elif k == 1:
                        sl = "".join(letters[j] for j in comb)  # "A"/"B"/"C"
                        f0 = float(lattice_dict["f(∅)"])
                        fA = float(lattice_dict[f"f({sl})"])
                        mu_cols = {f"μ({sl})": (fA - f0)}
                        resh_cols = reshAP_from_mu_recursive(mu_cols, sl)

                # row
                row = {
                    "Feature subset": label,
                    "n_features": k,
                    "R2 (subset test)": r2,
                    "RMSE (subset test)": rmse,
                    "Baseline f(∅)": baseline,
                    "Prediction f(S)": prediction,
                    "φ A": phi_cols[0] if m>=1 else "",
                    "φ B": phi_cols[1] if m>=2 else "",
                    "φ C": phi_cols[2] if m>=3 else "",
                    "f(∅)": lattice_dict.get("f(∅)", ""),
                    "f(A)": lattice_dict.get("f(A)", ""),
                    "f(B)": lattice_dict.get("f(B)", ""),
                    "f(AB)": lattice_dict.get("f(AB)", ""),
                }
                if m == 3:
                    row.update({
                        "f(C)": lattice_dict.get("f(C)", ""),
                        "f(AC)": lattice_dict.get("f(AC)", ""),
                        "f(BC)": lattice_dict.get("f(BC)", ""),
                        "f(ABC)": lattice_dict.get("f(ABC)", ""),
                    })
                for key in ["μ(A)","μ(B)","μ(C)","μ(AB)","μ(AC)","μ(BC)","μ(ABC)"]:
                    row[key] = mu_cols.get(key, "")
                for key in ["ReSHAP A","ReSHAP B","ReSHAP C"]:
                    row[key] = resh_cols.get(key, "")

                rows.append(row)

            # display table for this point + model
            base_cols = ["Feature subset","n_features","R2 (subset test)","RMSE (subset test)",
                         "Baseline f(∅)","Prediction f(S)","φ A","φ B"]
            if m==3: base_cols += ["φ C"]
            lattice_cols = ["f(∅)","f(A)","f(B)","f(AB)"]
            if m==3: lattice_cols += ["f(C)","f(AC)","f(BC)","f(ABC)"]
            mu_cols_all = ["μ(A)","μ(B)","μ(AB)"]
            if m==3: mu_cols_all = ["μ(A)","μ(B)","μ(C)","μ(AB)","μ(AC)","μ(BC)","μ(ABC)"]
            resh_cols_all = ["ReSHAP A","ReSHAP B"]
            if m==3: resh_cols_all += ["ReSHAP C"]

            ordered_cols = base_cols + lattice_cols + mu_cols_all + resh_cols_all
            tbl = pd.DataFrame(rows)[ordered_cols]
            
            # --- NEW: compute P, R, P/R ---
            metrics = compute_P_R_metrics(tbl)
            #model_metrics[mn].append(metrics)

            print(f"\n=== Subset predictions + SHAP + μ + ReSHAP — {mn} | Test idx {idx} ===")
            
            # Raw f-values
            print(f"f_AB(S)   = {metrics['f_AB(S)']}")
            print(f"f_AB(A)   = {metrics['f_AB(A)']}")
            print(f"f_ABC(S)  = {metrics['f_ABC(S)']}")
            print(f"f_ABC(A)  = {metrics['f_ABC(A)']}")
            
            # SHAP ψ terms + denominators
            print(f"|psi_AB(A)|    = {metrics['psi_AB(A)']}")
            print(f"|f_AB(S)-f_A|  = {metrics['den_AB']}")
            print(f"|psi_ABC(A)|   = {metrics['psi_ABC(A)']}")
            print(f"|f_ABC(S)-f_A| = {metrics['den_ABC']}")
            
            # ReSHAP terms
            print(f"|ReSHAP_AB(A)|  = {metrics['ReSHAP_AB(A)']}")
            print(f"|ReSHAP_ABC(A)| = {metrics['ReSHAP_ABC(A)']}")
            
            # Final metrics
            print(f"\nP   = {metrics['P']}")
            print(f"R   = {metrics['R']}")
            print(f"P/R = {metrics['P/R']}")
            
            display(tbl)

            # Build the ordered table (required to extract the values),
            # but do not display it per point
            ordered_cols = base_cols + lattice_cols + mu_cols_all + resh_cols_all
            tbl = pd.DataFrame(rows)[ordered_cols]
            
            # Compute P/R metrics for this (model, test point) and collect
            metrics = compute_P_R_metrics(tbl)
            model_metrics[mn].append(metrics)
    # ===== Overall Summary: mean of |P/R| across all chosen points =====
    summary_rows = []
    for mn, vals in model_metrics.items():
        # take absolute of per-point ratios, then average
        PR_abs_list = []
        for v in vals:
            pr = v.get("P/R", np.nan)
            if np.isfinite(pr):
                PR_abs_list.append(abs(pr))
    
        mean_abs_PR = float(np.mean(PR_abs_list)) if PR_abs_list else np.nan
        n_pts = len(vals)  # should match N_RANDOM_POINTS if all valid
    
        summary_rows.append({
            "Model": mn,
            "mean(|P/R|)": mean_abs_PR,
            "n_points": n_pts
        })
    
    summary_df = pd.DataFrame(summary_rows).set_index("Model")
    print("\n================ Overall Summary (averaged over test points) ================")
    display(summary_df[["mean(|P/R|)", "n_points"]])

# ============================================================
# 6) Settings & Run (edit here, then run this cell)
# ============================================================

SETTINGS = dict(
    # Data slice / feature modes:
    MODE="dup_qual",           # 'two' | 'dup_grliv' | 'dup_qual' | 'totrms'

    # Baseline used inside each subset for f(∅):
    BASELINE="mean",           # 'mean' | 'median' | 'zero'

    # Randomness & test row:
    RANDOM_STATE=42,
    PICK_RANDOM_POINT=True,    # True → draw a random test row (with the seed)
    N_RANDOM_POINTS=100,         # ← NEW: number of distinct random test points to assess
    CUSTOM_TEST_INDEX=None,    # if PICK_RANDOM_POINT=False, use this test index (must be in X_te)

    # Which models to run:
    USE_LINEAR=False,
    USE_MLP=True,
    USE_LGB=False,              # will only run if LightGBM is installed
    USE_XGB=False,              # will only run if XGBoost is installed

    # MLP hyperparameters (used if USE_MLP=True):
    MLP_hidden=(64, 32),
    MLP_max_iter=5000,
    MLP_early_stopping=True,
    MLP_n_iter_no_change=20,
    MLP_tol=1e-4,
    MLP_lr=1e-3,
    MLP_alpha=1e-4,

    # Warnings:
    SILENCE_MLP_WARNINGS=True
)

# ---- Go! (expects df_ames to already be in memory) ----
run_pipeline(SETTINGS)

Mode: dup_qual | Features (A,B,C): [('A', 'Gr Liv Area'), ('B', 'Overall Qual'), ('C', 'Overall Qual (DUP)')]
Evaluating 100 test point(s): [1093, 1032, 2783, 411, 1385, 1577, 1427, 518, 2181, 1880, 961, 1476, 1423, 1398, 2212, 96, 1080, 2044, 1397, 2786, 2233, 2684, 2359, 2227, 781, 2351, 1873, 1337, 2550, 211, 1557, 1910, 881, 554, 1462, 87, 2652, 478, 1505, 1283, 485, 134, 2605, 56, 282, 2240, 2111, 581, 1189, 70, 1878, 259, 495, 1506, 2841, 1608, 2916, 1691, 2394, 188, 1117, 555, 175, 366, 456, 506, 1538, 2381, 1820, 1600, 1486, 1221, 651, 1465, 120, 1058, 2659, 755, 1226, 2873, 2554, 1364, 291, 2662, 2834, 1264, 218, 2028, 2726, 1756, 856, 2718, 283, 2525, 527, 2421, 1509, 1662, 594, 1362]

──────── Test point index: 1093 | True SalePrice: 226750 ────────

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 1093 ===
f_AB(S)   = 226477.06687724203
f_AB(A)   = 170073.33312346545
f_ABC(S)  = 225058.58968850703
f_ABC(A)  = 166505.12773251915
|psi_AB(A)|    = 21867.743725858847
|f_AB(S)-f_A|  = 56403.733753776585
|psi_ABC(A)|   = 24436.11332120607
|f_ABC(S)-f_A| = 58553.46195598788
|ReSHAP_AB(A)|  = 21867.743725858843
|ReSHAP_ABC(A)| = 24414.61874371478

P   = -0.029629663181037713
R   = -0.029262569984952302
P/R = 1.0125448036954436

──────── Test point index: 1032 | True SalePrice: 306000 ────────

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 1032 ===
f_AB(S)   = 293320.0938323849
f_AB(A)   = 170073.33312346545
f_ABC(S)  = 303870.4711899712
f_ABC(A)  = 166505.12773251915
|psi_AB(A)|    = 49104.5775969823
|f_AB(S)-f_A|  = 123246.76070891946
|psi_ABC(A)|   = 52128.01218914255
|f_ABC(S)-f_A| = 137365.34345745205
|ReSHAP_AB(A)|  = 48532.66851551276
|ReSHAP_ABC(A)| = 52098.50042511663

P   = 0.01894044145569085
R   = 0.014514924951027519
P/R = 1.3048942050747596

──────── Test point index: 2783 | True SalePrice: 174000 ────────

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 2783 ===
f_AB(S)   = 156142.95321827102
f_AB(A)   = 170073.33312346545
f_ABC(S)  = 151298.1784716801
f_ABC(A)  = 166505.12773251915
|psi_AB(A)|    = 11552.38879976097
|f_AB(S)-f_A|  = 13930.379905194422
|psi_ABC(A)|   = 12868.449331459371
|f_ABC(S)-f_A| = 15206.949260839057
|ReSHAP_AB(A)|  = 11552.38879976098
|ReSHAP_ABC(A)| = 12868.449331459384

P   = -0.016927028361455765
R   = -0.016927028361455987
P/R = 0.9999999999999869

──────── Test point index: 411 | True SalePrice: 123000 ────────

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 411 ===
f_AB(S)   = 180620.26208389158
f_AB(A)   = 170073.33312346545
f_ABC(S)  = 173745.6972434084
f_ABC(A)  = 166505.12773251915
|psi_AB(A)|    = 23989.061067491624
|f_AB(S)-f_A|  = 10546.928960426128
|psi_ABC(A)|   = 26578.654253096665
|f_ABC(S)-f_A| = 7240.56951088924
|ReSHAP_AB(A)|  = 23989.061067491784
|ReSHAP_ABC(A)| = 28405.7825894105

P   = -1.3962893446798752
R   = -1.6486352870212708
P/R = 0.8469364665866576

──────── Test point index: 1385 | True SalePrice: 108959 ────────

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 1385 ===
f_AB(S)   = 102256.39318916843
f_AB(A)   = 170073.33312346545
f_ABC(S)  = 112880.82914882246
f_ABC(A)  = 166505.12773251915
|psi_AB(A)|    = 3900.93975408173
|f_AB(S)-f_A|  = 67816.93993429701
|psi_ABC(A)|   = 9482.786534445278
|f_ABC(S)-f_A| = 53624.29858369668
|ReSHAP_AB(A)|  = 10176.365975821107
|ReSHAP_ABC(A)| = 7717.1863247588835

P   = -0.11931588045816871
R   = 0.006144284757585827
P/R = -19.419002400704088

# ============================================================
# 6) Settings & Run (edit here, then run this cell)
# ============================================================

SETTINGS = dict(
    # Data slice / feature modes:
    MODE="totrms",           # 'two' | 'dup_grliv' | 'dup_qual' | 'totrms'

    # Baseline used inside each subset for f(∅):
    BASELINE="mean",           # 'mean' | 'median' | 'zero'

    # Randomness & test row:
    RANDOM_STATE=42,
    PICK_RANDOM_POINT=True,    # True → draw a random test row (with the seed)
    N_RANDOM_POINTS=100,         # ← NEW: number of distinct random test points to assess
    CUSTOM_TEST_INDEX=None,    # if PICK_RANDOM_POINT=False, use this test index (must be in X_te)

    # Which models to run:
    USE_LINEAR=False,
    USE_MLP=True,
    USE_LGB=False,              # will only run if LightGBM is installed
    USE_XGB=False,              # will only run if XGBoost is installed

    # MLP hyperparameters (used if USE_MLP=True):
    MLP_hidden=(64, 32),
    MLP_max_iter=5000,
    MLP_early_stopping=True,
    MLP_n_iter_no_change=20,
    MLP_tol=1e-4,
    MLP_lr=1e-3,
    MLP_alpha=1e-4,

    # Warnings:
    SILENCE_MLP_WARNINGS=True
)

# ---- Go! (expects df_ames to already be in memory) ----
run_pipeline(SETTINGS)

Mode: totrms | Features (A,B,C): [('A', 'Gr Liv Area'), ('B', 'Overall Qual'), ('C', 'TotRms AbvGrd')]
Evaluating 100 test point(s): [1093, 1032, 2783, 411, 1385, 1577, 1427, 518, 2181, 1880, 961, 1476, 1423, 1398, 2212, 96, 1080, 2044, 1397, 2786, 2233, 2684, 2359, 2227, 781, 2351, 1873, 1337, 2550, 211, 1557, 1910, 881, 554, 1462, 87, 2652, 478, 1505, 1283, 485, 134, 2605, 56, 282, 2240, 2111, 581, 1189, 70, 1878, 259, 495, 1506, 2841, 1608, 2916, 1691, 2394, 188, 1117, 555, 175, 366, 456, 506, 1538, 2381, 1820, 1600, 1486, 1221, 651, 1465, 120, 1058, 2659, 755, 1226, 2873, 2554, 1364, 291, 2662, 2834, 1264, 218, 2028, 2726, 1756, 856, 2718, 283, 2525, 527, 2421, 1509, 1662, 594, 1362]

──────── Test point index: 1093 | True SalePrice: 226750 ────────

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 1093 ===
f_AB(S)   = 226477.06687724203
f_AB(A)   = 170073.33312346545
f_ABC(S)  = 227390.7173978073
f_ABC(A)  = 166398.66206548671
|psi_AB(A)|    = 34535.99002791774
|f_AB(S)-f_A|  = 56403.733753776585
|psi_ABC(A)|   = 37189.86674034053
|f_ABC(S)-f_A| = 60992.0553323206
|ReSHAP_AB(A)|  = 34535.99002791774
|ReSHAP_ABC(A)| = 36638.014587376914

P   = 0.0025503530835897514
R   = 0.011598288096712372
P/R = 0.21989047541530457

──────── Test point index: 1032 | True SalePrice: 306000 ────────

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 1032 ===
f_AB(S)   = 293320.0938323849
f_AB(A)   = 170073.33312346545
f_ABC(S)  = 300504.07793133816
f_ABC(A)  = 166398.66206548671
|psi_AB(A)|    = 74142.18311193716
|f_AB(S)-f_A|  = 123246.76070891946
|psi_ABC(A)|   = 80964.56142862815
|f_ABC(S)-f_A| = 134105.41586585145
|ReSHAP_AB(A)|  = 74714.0921934067
|ReSHAP_ABC(A)| = 80125.87757910366

P   = -0.0021630870761817356
R   = 0.008731185671548292
P/R = -0.2477426500309617

──────── Test point index: 2783 | True SalePrice: 174000 ────────

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 2783 ===
f_AB(S)   = 156142.95321827102
f_AB(A)   = 170073.33312346545
f_ABC(S)  = 150684.75613624236
f_ABC(A)  = 166398.66206548671
|psi_AB(A)|    = 2377.9911054334516
|f_AB(S)-f_A|  = 13930.379905194422
|psi_ABC(A)|   = 2307.32369244559
|f_ABC(S)-f_A| = 15713.905929244356
|ReSHAP_AB(A)|  = 2377.991105433442
|ReSHAP_ABC(A)| = 2746.09175208566

P   = 0.023872164060555784
R   = -0.0040501145642329595
P/R = -5.894194764605842

──────── Test point index: 411 | True SalePrice: 123000 ────────

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 411 ===
f_AB(S)   = 180620.26208389158
f_AB(A)   = 170073.33312346545
f_ABC(S)  = 179699.29508187875
f_ABC(A)  = 166398.66206548671
|psi_AB(A)|    = 34535.99002791775
|f_AB(S)-f_A|  = 10546.928960426128
|psi_ABC(A)|   = 33361.450634454224
|f_ABC(S)-f_A| = 13300.633016392036
|ReSHAP_AB(A)|  = 34535.99002791791
|ReSHAP_ABC(A)| = 28061.353522773963

P   = 0.7662463800014185
R   = 1.1647309562729755
P/R = 0.657874143272822

──────── Test point index: 1385 | True SalePrice: 108959 ────────

=== Subset predictions + SHAP + μ + ReSHAP — MLP | Test idx 1385 ===
f_AB(S)   = 102256.39318916843
f_AB(A)   = 170073.33312346545
f_ABC(S)  = 109160.35647912517
f_ABC(A)  = 166398.66206548671
|psi_AB(A)|    = 63916.000180215284
|f_AB(S)-f_A|  = 67816.93993429701
|psi_ABC(A)|   = 45194.891384522685
|f_ABC(S)-f_A| = 57238.305586361545
|ReSHAP_AB(A)|  = 57640.57395847591
|ReSHAP_ABC(A)| = 47250.44913395942

P   = 0.15288668080009515
R   = 0.024439617256626023
P/R = 6.255690471529165

	Order	PID	MS SubClass	Lot Frontage	Lot Area	Overall Qual	Overall Cond	Year Built	Year Remod/Add	Mas Vnr Area	BsmtFin SF 1	BsmtFin SF 2	Bsmt Unf SF	Total Bsmt SF	1st Flr SF	2nd Flr SF	Low Qual Fin SF	Gr Liv Area	Bsmt Full Bath	Bsmt Half Bath	Full Bath	Half Bath	Bedroom AbvGr	Kitchen AbvGr	TotRms AbvGrd	Fireplaces	Garage Yr Blt	Garage Cars	Garage Area	Wood Deck SF	Open Porch SF	Enclosed Porch	3Ssn Porch	Screen Porch	Pool Area	Misc Val	Mo Sold	Yr Sold	SalePrice
count	2930.00000	2.930000e+03	2930.000000	2440.000000	2930.000000	2930.000000	2930.000000	2930.000000	2930.000000	2907.000000	2929.000000	2929.000000	2929.000000	2929.000000	2930.000000	2930.000000	2930.000000	2930.000000	2928.000000	2928.000000	2930.000000	2930.000000	2930.000000	2930.000000	2930.000000	2930.000000	2771.000000	2929.000000	2929.000000	2930.000000	2930.000000	2930.000000	2930.000000	2930.000000	2930.000000	2930.000000	2930.000000	2930.000000	2930.000000
mean	1465.50000	7.144645e+08	57.387372	69.224590	10147.921843	6.094881	5.563140	1971.356314	1984.266553	101.896801	442.629566	49.722431	559.262547	1051.614544	1159.557679	335.455973	4.676792	1499.690444	0.431352	0.061134	1.566553	0.379522	2.854266	1.044369	6.443003	0.599317	1978.132443	1.766815	472.819734	93.751877	47.533447	23.011604	2.592491	16.002048	2.243345	50.635154	6.216041	2007.790444	180796.060068
std	845.96247	1.887308e+08	42.638025	23.365335	7880.017759	1.411026	1.111537	30.245361	20.860286	179.112611	455.590839	169.168476	439.494153	440.615067	391.890885	428.395715	46.310510	505.508887	0.524820	0.245254	0.552941	0.502629	0.827731	0.214076	1.572964	0.647921	25.528411	0.760566	215.046549	126.361562	67.483400	64.139059	25.141331	56.087370	35.597181	566.344288	2.714492	1.316613	79886.692357
min	1.00000	5.263011e+08	20.000000	21.000000	1300.000000	1.000000	1.000000	1872.000000	1950.000000	0.000000	0.000000	0.000000	0.000000	0.000000	334.000000	0.000000	0.000000	334.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	2.000000	0.000000	1895.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	1.000000	2006.000000	12789.000000
25%	733.25000	5.284770e+08	20.000000	58.000000	7440.250000	5.000000	5.000000	1954.000000	1965.000000	0.000000	0.000000	0.000000	219.000000	793.000000	876.250000	0.000000	0.000000	1126.000000	0.000000	0.000000	1.000000	0.000000	2.000000	1.000000	5.000000	0.000000	1960.000000	1.000000	320.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	4.000000	2007.000000	129500.000000
50%	1465.50000	5.354536e+08	50.000000	68.000000	9436.500000	6.000000	5.000000	1973.000000	1993.000000	0.000000	370.000000	0.000000	466.000000	990.000000	1084.000000	0.000000	0.000000	1442.000000	0.000000	0.000000	2.000000	0.000000	3.000000	1.000000	6.000000	1.000000	1979.000000	2.000000	480.000000	0.000000	27.000000	0.000000	0.000000	0.000000	0.000000	0.000000	6.000000	2008.000000	160000.000000
75%	2197.75000	9.071811e+08	70.000000	80.000000	11555.250000	7.000000	6.000000	2001.000000	2004.000000	164.000000	734.000000	0.000000	802.000000	1302.000000	1384.000000	703.750000	0.000000	1742.750000	1.000000	0.000000	2.000000	1.000000	3.000000	1.000000	7.000000	1.000000	2002.000000	2.000000	576.000000	168.000000	70.000000	0.000000	0.000000	0.000000	0.000000	0.000000	8.000000	2009.000000	213500.000000
max	2930.00000	1.007100e+09	190.000000	313.000000	215245.000000	10.000000	9.000000	2010.000000	2010.000000	1600.000000	5644.000000	1526.000000	2336.000000	6110.000000	5095.000000	2065.000000	1064.000000	5642.000000	3.000000	2.000000	4.000000	2.000000	8.000000	3.000000	15.000000	4.000000	2207.000000	5.000000	1488.000000	1424.000000	742.000000	1012.000000	508.000000	576.000000	800.000000	17000.000000	12.000000	2010.000000	755000.000000

	Feature subset	n_features	R2 (subset test)	RMSE (subset test)	Baseline f(∅)	Prediction f(S)	φ A	φ B	f(∅)	f(A)	f(B)	f(AB)	μ(A)	μ(B)	μ(AB)	ReSHAP A	ReSHAP B
0	nothing	0	-0.015283	90222.372400	178582.207765	178582.207765			178582.207765
1	A	1	0.523397	61815.726690	178582.207765	221182.120972	42599.913207		178582.207765	221182.120972			42599.913207			42599.913207
2	B	1	0.651246	52878.678358	178582.207765	219911.155344		41328.947579	178582.207765		219911.155344			41328.947579			41328.947579
3	AB	2	0.743635	45336.691057	178582.207765	231715.622003	22524.873238	30608.541001	178582.207765	201107.081002	209190.748766	231715.622003	22524.873238	30608.541001	-0.0	22524.873238	30608.541001

	Feature subset	n_features	R2 (subset test)	RMSE (subset test)	Baseline f(∅)	Prediction f(S)	φ A	φ B	f(∅)	f(A)	f(B)	f(AB)	μ(A)	μ(B)	μ(AB)	ReSHAP A	ReSHAP B
0	nothing	0	-0.015283	90222.372400	178582.207765	178582.207765			178582.207765
1	A	1	0.526650	61604.437299	178607.720643	222692.229667	44084.509025		178607.720643	222692.229667			44084.509025			44084.509025
2	B	1	0.653707	52691.751258	167830.652805	213037.816252		45207.163448	167830.652805		213037.816252			45207.163448			45207.163448
3	AB	2	0.749226	44839.624931	170073.333123	226477.066877	21867.743726	34535.990028	170073.333123	191941.076849	204609.323151	226477.066877	21867.743726	34535.990028	0.0	21867.743726	34535.990028

	Feature subset	n_features	R2 (subset test)	RMSE (subset test)	Baseline f(∅)	Prediction f(S)	φ A	φ B	f(∅)	f(A)	f(B)	f(AB)	μ(A)	μ(B)	μ(AB)	ReSHAP A	ReSHAP B
0	nothing	0	-0.015283	90222.372400	178582.207765	178582.207765			178582.207765
1	A	1	0.494836	63641.022042	207590.130539	198246.330716	-9343.799823		207590.130539	198246.330716			-9343.799823			-9343.799823
2	B	1	0.725528	46910.500909	162512.760008	204048.490253		41535.730245	162512.760008		204048.490253			41535.730245			41535.730245
3	AB	2	0.809248	39107.119385	182789.632430	200534.205454	-13995.525233	31740.098257	182789.632430	183753.304113	229488.927603	200534.205454	963.671683	46699.295173	-29918.393833	358.767901	17385.805123

	Feature subset	n_features	R2 (subset test)	RMSE (subset test)	Baseline f(∅)	Prediction f(S)	φ A	φ B	f(∅)	f(A)	f(B)	f(AB)	μ(A)	μ(B)	μ(AB)	ReSHAP A	ReSHAP B
0	nothing	0	-0.015283	90222.372400	178582.207765	178582.207765			178582.207765
1	A	1	0.462856	65624.546618	195086.234375	187762.781250	-7323.453125		195086.234375	187762.78125			-7323.453125			-7323.453125
2	B	1	0.725640	46900.883370	162447.125000	204150.109375		41702.984375	162447.125000		204150.109375			41702.984375			41702.984375
3	AB	2	0.790517	40982.155774	172155.718750	201395.015625	657.195312	28582.101562	172155.718750	172812.921875	200737.828125	201395.015625	657.203125	28582.109375	-0.015625	657.202774	28582.094101

ReSHAP: Experimental Code¶

Chapter 1: Analysis Dataset¶

Functions Definitions¶

Investigate 2 Features¶

Duplicate "Overall Quality (B == C)"¶

Duplicate "Gr Liv Area (A == C)"¶

Insert TotRms (Add correlated C)"¶

P / R Investigation¶

	Gr Liv Area	Overall Qual	SalePrice
SalePrice	0.706780	0.799262	1.000000
Overall Qual	0.570556	1.000000	0.799262
Gr Liv Area	1.000000	0.570556	0.706780
Garage Cars	0.488829	0.599545	0.647877
Garage Area	0.484892	0.563503	0.640401
Total Bsmt SF	0.444675	0.547294	0.632280
1st Flr SF	0.562166	0.477837	0.621676
Year Built	0.241726	0.597027	0.558426
Full Bath	0.630321	0.522263	0.545604
Year Remod/Add	0.316855	0.569609	0.532974
Garage Yr Blt	0.272848	0.570569	0.526965
Mas Vnr Area	0.403611	0.429418	0.508285
TotRms AbvGrd	0.807772	0.380693	0.495474
Fireplaces	0.454924	0.393007	0.474558
BsmtFin SF 1	0.209633	0.284118	0.432914