Skip to content

EBM benchmark with skorecard

This benchmark was adjusted from this notebook

# To run benchmark script, you will need to install XGBoost 
# (pip install XGBoost)

import numpy as np
import pandas as pd
from sklearn.datasets import load_breast_cancer
import warnings
warnings.filterwarnings("ignore")

def load_breast_data():
    breast = load_breast_cancer()
    feature_names = list(breast.feature_names)
    X, y = pd.DataFrame(breast.data, columns=feature_names), breast.target
    dataset = {
        'problem': 'classification',
        'full': {
            'X': X,
            'y': y,
        },
    }
    return dataset


def load_adult_data():
    df = pd.read_csv(
        "https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data",
        header=None)
    df.columns = [
        "Age", "WorkClass", "fnlwgt", "Education", "EducationNum",
        "MaritalStatus", "Occupation", "Relationship", "Race", "Gender",
        "CapitalGain", "CapitalLoss", "HoursPerWeek", "NativeCountry", "Income"
    ]
    train_cols = df.columns[0:-1]
    label = df.columns[-1]
    X_df = df[train_cols]
    y_df = df[label]

    dataset = {
        'problem': 'classification',
        'full': {
            'X': X_df,
            'y': y_df,
        },
    }

    return dataset

def load_heart_data():
    # https://www.kaggle.com/ronitf/heart-disease-uci
    df = pd.read_csv(r'heart.csv')
    train_cols = df.columns[0:-1]
    label = df.columns[-1]
    X_df = df[train_cols]
    y_df = df[label]
    dataset = {
        'problem': 'classification',
        'full': {
            'X': X_df,
            'y': y_df,
        },
    }

    return dataset


def load_credit_data():
    # https://www.kaggle.com/mlg-ulb/creditcardfraud
    df = pd.read_csv(r'creditcard.csv')
    train_cols = df.columns[0:-1]
    label = df.columns[-1]
    X_df = df[train_cols]
    y_df = df[label]
    dataset = {
        'problem': 'classification',
        'full': {
            'X': X_df,
            'y': y_df,
        },
    }

    return dataset


def load_telco_churn_data():
    # https://www.kaggle.com/blastchar/telco-customer-churn
    df = pd.read_csv(r'WA_Fn-UseC_-Telco-Customer-Churn.csv')
    train_cols = df.columns[1:-1] # First column is an ID
    label = df.columns[-1]
    X_df = df[train_cols]
    y_df = df[label] # 'Yes, No'
    dataset = {
        'problem': 'classification',
        'full': {
            'X': X_df,
            'y': y_df,
        },
    }

    return dataset
from sklearn.preprocessing import OneHotEncoder, FunctionTransformer, StandardScaler
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.model_selection import StratifiedShuffleSplit, cross_validate
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import OrdinalEncoder

from xgboost import XGBClassifier

from sklearn.linear_model import LogisticRegression

from interpret.glassbox import ExplainableBoostingClassifier

from skorecard import Skorecard

from optbinning import BinningProcess
from optbinning import Scorecard


def format_n(x):
    return "{0:.3f}".format(x)

def process_model(clf, name, X, y, n_splits=3):
    # Evaluate model
    ss = StratifiedShuffleSplit(n_splits=n_splits, test_size=0.25, random_state=1337)
    scores = cross_validate(
        clf, X, y, scoring='roc_auc', cv=ss,
        n_jobs=-1, return_estimator=True
    )

    record = dict()
    record['model_name'] = name
    record['fit_time_mean'] = format_n(np.mean(scores['fit_time']))
    record['fit_time_std'] = format_n(np.std(scores['fit_time']))
    record['test_score_mean'] = format_n(np.mean(scores['test_score']))
    record['test_score_std'] = format_n(np.std(scores['test_score']))

    return record



def benchmark_models(dataset_name, X, y, ct=None, n_splits=3, random_state=1337):
    if ct is None:
        is_cat = np.array([dt.kind == 'O' for dt in X.dtypes])
        cat_cols = X.columns.values[is_cat]
        num_cols = X.columns.values[~is_cat]

        cat_ohe_step = ('ohe', OneHotEncoder(sparse=False,
                                             handle_unknown='ignore'))

        cat_pipe = Pipeline([cat_ohe_step])
        num_pipe = Pipeline([('identity', FunctionTransformer())])
        transformers = [
            ('cat', cat_pipe, cat_cols),
            ('num', num_pipe, num_cols)
        ]
        ct = ColumnTransformer(transformers=transformers)

        cat_ord_step = ('ord_enc', OrdinalEncoder())
        cat_pipe = Pipeline([cat_ord_step])
        transformers = [
            ('cat', cat_pipe, cat_cols),
            ('num', num_pipe, num_cols)
        ]
        ot = ColumnTransformer(transformers=transformers)

    records = []

    summary_record = {}
    summary_record['dataset_name'] = dataset_name
    print()
    print('-' * 78)
    print(dataset_name)
    print('-' * 78)
    print(summary_record)
    print()

    pipe = Pipeline([
        ('ct', ct),
        ('std', StandardScaler()),
        ('lr', LogisticRegression(random_state=random_state)),
    ])
    record = process_model(pipe, 'lr_ohe', X, y, n_splits=n_splits)
    print(record)
    record.update(summary_record)
    records.append(record)

    pipe = Pipeline([
        ('ot', ot),
        ('std', StandardScaler()),
        ('lr', LogisticRegression(max_iter=7000, random_state=random_state)),
    ])
    record = process_model(pipe, 'lr_ordinal', X, y, n_splits=n_splits)
    print(record)
    record.update(summary_record)
    records.append(record)

    # Skorecard
    skorecard = Skorecard()
    record = process_model(skorecard, 'skorecard', X, y, n_splits=n_splits)
    print(record)
    record.update(summary_record)
    records.append(record)

    pipe = Pipeline([
        ('ct', ct),
        # n_estimators updated from 10 to 100 due to sci-kit defaults changing in future versions
        ('rf-100', RandomForestClassifier(n_estimators=100, n_jobs=-1, random_state=random_state)),
    ])
    record = process_model(pipe, 'rf-100', X, y, n_splits=n_splits)
    print(record)
    record.update(summary_record)
    records.append(record)

    pipe = Pipeline([
        ('ct', ct),
        ('xgb', XGBClassifier(random_state=random_state, eval_metric='logloss')),
    ])
    record = process_model(pipe, 'xgb', X, y, n_splits=n_splits)
    print(record)
    record.update(summary_record)
    records.append(record)

    # No pipeline needed due to EBM handling string datatypes
    ebm_inter = ExplainableBoostingClassifier(n_jobs=-1, random_state=random_state)
    record = process_model(ebm_inter, 'ebm', X, y, n_splits=n_splits)
    print(record)
    record.update(summary_record)
    records.append(record)

    return records
results = []
n_splits = 3
from skorecard.datasets import load_uci_credit_card

X,y = load_uci_credit_card(return_X_y=True)
result = benchmark_models('UCI-creditcard', X, y, n_splits=n_splits)
results.append(result)

------------------------------------------------------------------------------
UCI-creditcard
------------------------------------------------------------------------------
{'dataset_name': 'UCI-creditcard'}

{'model_name': 'lr_ohe', 'fit_time_mean': '0.009', 'fit_time_std': '0.000', 'test_score_mean': '0.621', 'test_score_std': '0.023'}
{'model_name': 'lr_ordinal', 'fit_time_mean': '0.008', 'fit_time_std': '0.001', 'test_score_mean': '0.621', 'test_score_std': '0.023'}
{'model_name': 'skorecard', 'fit_time_mean': '1.489', 'fit_time_std': '0.044', 'test_score_mean': '0.627', 'test_score_std': '0.018'}
{'model_name': 'rf-100', 'fit_time_mean': '0.326', 'fit_time_std': '0.051', 'test_score_mean': '0.588', 'test_score_std': '0.013'}
{'model_name': 'xgb', 'fit_time_mean': '0.957', 'fit_time_std': '0.114', 'test_score_mean': '0.596', 'test_score_std': '0.005'}
{'model_name': 'ebm', 'fit_time_mean': '1.219', 'fit_time_std': '0.151', 'test_score_mean': '0.644', 'test_score_std': '0.012'}

dataset = load_breast_data()
result = benchmark_models('breast-cancer', dataset['full']['X'], dataset['full']['y'], n_splits=n_splits)
results.append(result)

------------------------------------------------------------------------------
breast-cancer
------------------------------------------------------------------------------
{'dataset_name': 'breast-cancer'}

{'model_name': 'lr_ohe', 'fit_time_mean': '0.014', 'fit_time_std': '0.001', 'test_score_mean': '0.994', 'test_score_std': '0.006'}
{'model_name': 'lr_ordinal', 'fit_time_mean': '0.015', 'fit_time_std': '0.001', 'test_score_mean': '0.994', 'test_score_std': '0.006'}
{'model_name': 'skorecard', 'fit_time_mean': '13.034', 'fit_time_std': '0.868', 'test_score_mean': '0.996', 'test_score_std': '0.004'}
{'model_name': 'rf-100', 'fit_time_mean': '0.253', 'fit_time_std': '0.002', 'test_score_mean': '0.992', 'test_score_std': '0.009'}
{'model_name': 'xgb', 'fit_time_mean': '0.196', 'fit_time_std': '0.014', 'test_score_mean': '0.992', 'test_score_std': '0.010'}
{'model_name': 'ebm', 'fit_time_mean': '5.866', 'fit_time_std': '1.070', 'test_score_mean': '0.995', 'test_score_std': '0.006'}

dataset = load_adult_data()
result = benchmark_models('adult', dataset['full']['X'], dataset['full']['y'], n_splits=n_splits)
results.append(result)
# 0.888

------------------------------------------------------------------------------
adult
------------------------------------------------------------------------------
{'dataset_name': 'adult'}

{'model_name': 'lr_ohe', 'fit_time_mean': '0.527', 'fit_time_std': '0.032', 'test_score_mean': '0.906', 'test_score_std': '0.003'}
{'model_name': 'lr_ordinal', 'fit_time_mean': '0.131', 'fit_time_std': '0.003', 'test_score_mean': '0.855', 'test_score_std': '0.002'}
{'model_name': 'skorecard', 'fit_time_mean': '8.047', 'fit_time_std': '0.025', 'test_score_mean': '0.888', 'test_score_std': '0.004'}
{'model_name': 'rf-100', 'fit_time_mean': '2.445', 'fit_time_std': '0.008', 'test_score_mean': '0.903', 'test_score_std': '0.002'}
{'model_name': 'xgb', 'fit_time_mean': '14.403', 'fit_time_std': '0.403', 'test_score_mean': '0.927', 'test_score_std': '0.001'}
{'model_name': 'ebm', 'fit_time_mean': '51.093', 'fit_time_std': '1.137', 'test_score_mean': '0.928', 'test_score_std': '0.002'}

dataset = load_telco_churn_data()
result = benchmark_models('telco_churn', dataset['full']['X'], dataset['full']['y'], n_splits=n_splits)
results.append(result)

------------------------------------------------------------------------------
telco_churn
------------------------------------------------------------------------------
{'dataset_name': 'telco_churn'}

{'model_name': 'lr_ohe', 'fit_time_mean': '9.091', 'fit_time_std': '0.030', 'test_score_mean': '0.809', 'test_score_std': '0.014'}
{'model_name': 'lr_ordinal', 'fit_time_mean': '0.064', 'fit_time_std': '0.002', 'test_score_mean': 'nan', 'test_score_std': 'nan'}
{'model_name': 'skorecard', 'fit_time_mean': '4.672', 'fit_time_std': '0.069', 'test_score_mean': '0.764', 'test_score_std': '0.014'}
{'model_name': 'rf-100', 'fit_time_mean': '7.118', 'fit_time_std': '0.016', 'test_score_mean': '0.824', 'test_score_std': '0.002'}
{'model_name': 'xgb', 'fit_time_mean': '104.738', 'fit_time_std': '0.969', 'test_score_mean': '0.825', 'test_score_std': '0.003'}
{'model_name': 'ebm', 'fit_time_mean': '36.052', 'fit_time_std': '3.779', 'test_score_mean': '0.852', 'test_score_std': '0.004'}

dataset = load_heart_data()
result = benchmark_models('heart', dataset['full']['X'], dataset['full']['y'], n_splits=n_splits)
results.append(result)

------------------------------------------------------------------------------
heart
------------------------------------------------------------------------------
{'dataset_name': 'heart'}

{'model_name': 'lr_ohe', 'fit_time_mean': '0.007', 'fit_time_std': '0.001', 'test_score_mean': '0.895', 'test_score_std': '0.030'}
{'model_name': 'lr_ordinal', 'fit_time_mean': '0.007', 'fit_time_std': '0.000', 'test_score_mean': '0.895', 'test_score_std': '0.030'}
{'model_name': 'skorecard', 'fit_time_mean': '2.148', 'fit_time_std': '0.013', 'test_score_mean': '0.911', 'test_score_std': '0.015'}
{'model_name': 'rf-100', 'fit_time_mean': '0.241', 'fit_time_std': '0.003', 'test_score_mean': '0.890', 'test_score_std': '0.008'}
{'model_name': 'xgb', 'fit_time_mean': '0.318', 'fit_time_std': '0.035', 'test_score_mean': '0.851', 'test_score_std': '0.018'}
{'model_name': 'ebm', 'fit_time_mean': '1.925', 'fit_time_std': '0.392', 'test_score_mean': '0.906', 'test_score_std': '0.011'}

records = [item for result in results for item in result]
record_df = pd.DataFrame.from_records(records)[['dataset_name', 'model_name', 'test_score_mean', 'test_score_std']]
record_df = record_df.sort_values(['dataset_name','test_score_mean'], ascending=False)
print(record_df[record_df['model_name'].isin(['lr_ohe','lr_ordinal','rf-100','skorecard','xgb'])].drop(['test_score_std'],axis=1).to_markdown(tablefmt="github",showindex=False))
| dataset_name   | model_name   |   test_score_mean |
|----------------|--------------|-------------------|
| telco_churn    | lr_ordinal   |           nan     |
| telco_churn    | xgb          |             0.825 |
| telco_churn    | rf-100       |             0.824 |
| telco_churn    | lr_ohe       |             0.809 |
| telco_churn    | skorecard    |             0.764 |
| heart          | skorecard    |             0.911 |
| heart          | lr_ohe       |             0.895 |
| heart          | lr_ordinal   |             0.895 |
| heart          | rf-100       |             0.89  |
| heart          | xgb          |             0.851 |
| breast-cancer  | skorecard    |             0.996 |
| breast-cancer  | lr_ohe       |             0.994 |
| breast-cancer  | lr_ordinal   |             0.994 |
| breast-cancer  | rf-100       |             0.992 |
| breast-cancer  | xgb          |             0.992 |
| adult          | xgb          |             0.927 |
| adult          | lr_ohe       |             0.906 |
| adult          | rf-100       |             0.903 |
| adult          | skorecard    |             0.888 |
| adult          | lr_ordinal   |             0.855 |
| UCI-creditcard | skorecard    |             0.627 |
| UCI-creditcard | lr_ohe       |             0.621 |
| UCI-creditcard | lr_ordinal   |             0.621 |
| UCI-creditcard | xgb          |             0.596 |
| UCI-creditcard | rf-100       |             0.588 |


Last update: 2021-11-24
Back to top