Breast Cancer Classification — Improved ML Comparison Project

Usage:
    python breast_cancer_ml.py
    python breast_cancer_ml.py your_data.csv

CSV requirements:
    - A target column named "target" or "class"
    - All feature columns should be numeric
    - Missing numeric values are imputed within each model pipeline

import sys
import warnings
from pathlib import Path

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import (
    train_test_split,
    StratifiedKFold,
    cross_validate,
)
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.metrics import (
    accuracy_score,
    precision_score,
    recall_score,
    f1_score,
    roc_auc_score,
    roc_curve,
    confusion_matrix,
    classification_report,
    make_scorer,
)
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB

RANDOM_STATE = 42
TEST_SIZE = 0.20
CV_FOLDS = 5
OUTPUT_DIR = Path("results")

def load_data(path=None):
    """Load the built-in breast cancer dataset or a labeled CSV."""
    if path is None:
        dataset = load_breast_cancer()
        X = pd.DataFrame(dataset.data, columns=dataset.feature_names)
        y = pd.Series(dataset.target, name="target")
        class_names = list(dataset.target_names)
        print("Dataset: scikit-learn Breast Cancer Wisconsin dataset")
    else:
        csv_path = Path(path)
        if not csv_path.is_file():
            raise FileNotFoundError(f"CSV file not found: {csv_path}")

        df = pd.read_csv(csv_path)
        target_col = next(
            (col for col in ("target", "class") if col in df.columns),
            None,
        )
        if target_col is None:
            raise ValueError(
                'CSV must contain a target column named "target" or "class".'
            )
        if df.empty:
            raise ValueError("The CSV file contains no rows.")

        X = df.drop(columns=[target_col]).copy()
        y = df[target_col].copy()
        class_names = None
        print(f"Dataset: {csv_path}")

    if X.shape[1] == 0:
        raise ValueError("No feature columns found.")
    if y.isna().any():
        raise ValueError("The target column contains missing values.")
    if y.nunique() != 2:
        raise ValueError(
            f"This project supports binary classification; found {y.nunique()} classes."
        )

    # Convert feature columns to numeric, rejecting unsupported text columns.
    for col in X.columns:
        X[col] = pd.to_numeric(X[col], errors="coerce")
    X = X.replace([np.inf, -np.inf], np.nan)

    all_missing = X.columns[X.isna().all()].tolist()
    if all_missing:
        raise ValueError(
            "These feature columns contain no usable numeric values: "
            + ", ".join(map(str, all_missing))
        )

    # Encode labels consistently as 0 and 1.
    encoder = LabelEncoder()
    y_encoded = pd.Series(encoder.fit_transform(y), index=y.index, name="target")
    if class_names is None:
        class_names = [str(value) for value in encoder.classes_]

    if y_encoded.value_counts().min() < CV_FOLDS:
        raise ValueError(
            f"Each class needs at least {CV_FOLDS} examples for {CV_FOLDS}-fold "
            "stratified cross-validation."
        )

    return X, y_encoded, class_names

def build_models():
    """Build comparable pipelines; preprocessing is fit inside each fold."""
    scaled_models = {
        "Logistic Regression": LogisticRegression(
            max_iter=5000, class_weight="balanced", random_state=RANDOM_STATE
        ),
        "SVM": SVC(
            probability=True, class_weight="balanced", random_state=RANDOM_STATE
        ),
        "K-Nearest Neighbors": KNeighborsClassifier(n_neighbors=5),
        "Gaussian Naive Bayes": GaussianNB(),
    }

    models = {}
    for name, estimator in scaled_models.items():
        models[name] = Pipeline(
            [
                ("imputer", SimpleImputer(strategy="median")),
                ("scaler", StandardScaler()),
                ("model", estimator),
            ]
        )

    # Tree models do not require feature scaling.
    models["Random Forest"] = Pipeline(
        [
            ("imputer", SimpleImputer(strategy="median")),
            (
                "model",
                RandomForestClassifier(
                    n_estimators=300,
                    class_weight="balanced",
                    random_state=RANDOM_STATE,
                    n_jobs=-1,
                ),
            ),
        ]
    )
    models["Gradient Boosting"] = Pipeline(
        [
            ("imputer", SimpleImputer(strategy="median")),
            (
                "model",
                GradientBoostingClassifier(
                    n_estimators=150,
                    learning_rate=0.05,
                    random_state=RANDOM_STATE,
                ),
            ),
        ]
    )
    return models

def positive_class_scores(model, X):
    """Return continuous scores for class 1, suitable for ROC-AUC."""
    if hasattr(model, "predict_proba"):
        probabilities = model.predict_proba(X)
        classes = model.named_steps["model"].classes_
        positive_index = list(classes).index(1)
        return probabilities[:, positive_index]

    scores = model.decision_function(X)
    classes = model.named_steps["model"].classes_
    return scores if list(classes)[-1] == 1 else -scores

def evaluate_models(models, X_train, X_test, y_train, y_test):
    results = {}
    for name, model in models.items():
        model.fit(X_train, y_train)
        y_pred = model.predict(X_test)
        y_score = positive_class_scores(model, X_test)

        results[name] = {
            "model": model,
            "accuracy": accuracy_score(y_test, y_pred),
            "precision": precision_score(y_test, y_pred, zero_division=0),
            "recall": recall_score(y_test, y_pred, zero_division=0),
            "f1": f1_score(y_test, y_pred, zero_division=0),
            "auc": roc_auc_score(y_test, y_score),
            "cm": confusion_matrix(y_test, y_pred, labels=[0, 1]),
            "y_score": y_score,
            "y_pred": y_pred,
        }
    return results

def cross_validate_models(models, X_train, y_train, folds=CV_FOLDS):
    """Evaluate multiple metrics on training data only."""
    cv = StratifiedKFold(
        n_splits=folds, shuffle=True, random_state=RANDOM_STATE
    )
    scoring = {
        "accuracy": "accuracy",
        "precision": make_scorer(precision_score, zero_division=0),
        "recall": make_scorer(recall_score, zero_division=0),
        "f1": make_scorer(f1_score, zero_division=0),
        "roc_auc": "roc_auc",
    }
    cv_results = {}
    for name, model in models.items():
        scores = cross_validate(
            model,
            X_train,
            y_train,
            cv=cv,
            scoring=scoring,
            n_jobs=-1,
            error_score="raise",
        )
        cv_results[name] = {
            metric: scores[f"test_{metric}"]
            for metric in scoring
        }
    return cv_results

def build_summary(results, cv_results):
    rows = []
    for name, result in results.items():
        row = {
            "Model": name,
            "Test Accuracy": result["accuracy"],
            "Test Precision": result["precision"],
            "Test Recall": result["recall"],
            "Test F1": result["f1"],
            "Test ROC-AUC": result["auc"],
        }
        if name in cv_results:
            row["CV ROC-AUC Mean"] = cv_results[name]["roc_auc"].mean()
            row["CV ROC-AUC Std"] = cv_results[name]["roc_auc"].std()
        rows.append(row)
    return pd.DataFrame(rows).sort_values("Test ROC-AUC", ascending=False)

def plot_roc_curves(results, y_test, output_dir):
    fig, ax = plt.subplots(figsize=(9, 7))
    for name, result in results.items():
        fpr, tpr, _ = roc_curve(y_test, result["y_score"])
        ax.plot(fpr, tpr, linewidth=2, label=f'{name} (AUC={result["auc"]:.3f})')
    ax.plot([0, 1], [0, 1], "k--", alpha=0.6, label="Random classifier")
    ax.set(
        xlabel="False Positive Rate",
        ylabel="True Positive Rate",
        title="ROC Curves — Test Set",
        xlim=(0, 1),
        ylim=(0, 1.02),
    )
    ax.legend(loc="lower right", fontsize=8)
    ax.grid(alpha=0.2)
    fig.tight_layout()
    fig.savefig(output_dir / "roc_curves.png", dpi=200, bbox_inches="tight")
    plt.close(fig)

def plot_confusion_matrices(results, output_dir, class_names):
    cols = 3
    rows = int(np.ceil(len(results) / cols))
    fig, axes = plt.subplots(rows, cols, figsize=(14, 4.2 * rows))
    axes = np.array(axes).reshape(-1)

    for ax, (name, result) in zip(axes, results.items()):
        sns.heatmap(
            result["cm"],
            annot=True,
            fmt="d",
            cmap="Blues",
            cbar=False,
            xticklabels=class_names,
            yticklabels=class_names,
            ax=ax,
        )
        ax.set_title(name)
        ax.set_xlabel("Predicted label")
        ax.set_ylabel("Actual label")

    for ax in axes[len(results):]:
        ax.axis("off")

    fig.suptitle("Confusion Matrices — Test Set", fontsize=15, y=1.01)
    fig.tight_layout()
    fig.savefig(
        output_dir / "confusion_matrices.png", dpi=200, bbox_inches="tight"
    )
    plt.close(fig)

def plot_metric_comparison(summary, output_dir):
    metric_columns = [
        "Test Accuracy", "Test Precision", "Test Recall", "Test F1", "Test ROC-AUC"
    ]
    plot_df = summary.set_index("Model")[metric_columns]
    ax = plot_df.plot(kind="bar", figsize=(13, 6), ylim=(0, 1.05), width=0.82)
    ax.set_title("Model Performance Comparison — Test Set")
    ax.set_ylabel("Score")
    ax.set_xlabel("")
    ax.tick_params(axis="x", rotation=25)
    ax.grid(axis="y", alpha=0.25)
    ax.legend(loc="lower right")
    plt.tight_layout()
    plt.savefig(
        output_dir / "model_comparison.png", dpi=200, bbox_inches="tight"
    )
    plt.close()

def print_summary(summary, cv_results):
    print("\n=== Test Set Performance (higher is better) ===")
    print(summary.round(4).to_string(index=False))

    print("\n=== Cross-Validation Results on Training Set (mean ± std) ===")
    for name, metrics in cv_results.items():
        print(f"\n{name}")
        for metric, values in metrics.items():
            print(f"  {metric:10s}: {values.mean():.4f} ± {values.std():.4f}")

def main():
    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
    sns.set_theme(style="whitegrid", context="notebook")

    csv_path = sys.argv[1] if len(sys.argv) > 1 else None

    try:
        X, y, class_names = load_data(csv_path)
        print(f"Rows: {len(X)} | Features: {X.shape[1]} | Classes: {class_names}")
        print(f"Missing feature values: {int(X.isna().sum().sum())}")

        X_train, X_test, y_train, y_test = train_test_split(
            X,
            y,
            test_size=TEST_SIZE,
            stratify=y,
            random_state=RANDOM_STATE,
        )

        models = build_models()
        cv_results = cross_validate_models(models, X_train, y_train)
        results = evaluate_models(models, X_train, X_test, y_train, y_test)

        summary = build_summary(results, cv_results)
        print_summary(summary, cv_results)

        summary.to_csv(OUTPUT_DIR / "model_metrics.csv", index=False)
        plot_roc_curves(results, y_test, OUTPUT_DIR)
        plot_confusion_matrices(results, OUTPUT_DIR, class_names)
        plot_metric_comparison(summary, OUTPUT_DIR)

        best_name = summary.iloc[0]["Model"]
        best_result = results[best_name]
        print(f"\nBest model by test ROC-AUC: {best_name}")
        print("\nClassification report for best model:")
        print(
            classification_report(
                y_test,
                best_result["y_pred"],
                labels=[0, 1],
                target_names=class_names,
                zero_division=0,
            )
        )
        print(f"Outputs saved in: {OUTPUT_DIR.resolve()}")
        print(" - model_metrics.csv")
        print(" - roc_curves.png")
        print(" - confusion_matrices.png")
        print(" - model_comparison.png")

    except (ValueError, FileNotFoundError, pd.errors.ParserError) as exc:
        print(f"Error: {exc}", file=sys.stderr)
        sys.exit(1)

if __name__ == "__main__":
    main()