In [1]:
import pandas as pd
pd.set_option('display.max_columns', 50)
import numpy as np
import os
os.chdir('D:\Data\Projects\Classification\Heart Disease')
import matplotlib.pyplot as plt
plt.style.use('Solarize_Light2')
plt.rcParams['font.size'] = 15
import seaborn as sns
from warnings import filterwarnings
filterwarnings('ignore')
from IPython.core.pylabtools import figsize
figsize(10, 10)
In [2]:
import plotly.offline as py
import plotly.graph_objs as go
In [3]:
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import f1_score, confusion_matrix, roc_auc_score, roc_curve, classification_report, accuracy_score
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn import model_selection
from sklearn.dummy import DummyClassifier
from sklearn.linear_model import LogisticRegression
In [4]:
df = pd.read_csv('heart.csv')
df.shape
Out[4]:
In [5]:
df.head(5)
Out[5]:
In [6]:
x = df.drop('target', axis=1)
y = df.target
x.shape, y.shape
Out[6]:
In [7]:
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)
X_train.shape, X_test.shape, y_train.shape, y_test.shape
Out[7]:
Standardisieren der Daten mit Standard Scaler¶
Mittelwert bei 0, Standardabweichung 1
Für einige der Algorithmen, wie Logistic Regression und Support Vector Machines, ist das Standardisieren wichtig, um optimale Ergebnisse zu erzielen.
Das Fitten geschieht ausschließlich auf X_train, transformiert werden dann X_train und X_test.
In [8]:
sc = StandardScaler()
sc.fit(X_train)
Out[8]:
In [9]:
X_train =sc.transform(X_train);
X_test =sc.transform(X_test);
Baseline Model¶
Das Dummy Model sagt immer den Mittelwert der Zielvariablen voraus
In [10]:
# Create dummy classifer
dummy = DummyClassifier(strategy='uniform', random_state=1)
# "Train" model
dummy.fit(X_train, y_train)
Out[10]:
In [11]:
# Get accuracy score
dummy.score(X_test, y_test)
Out[11]:
Baseline Logistic Regression¶
In [12]:
lr = LogisticRegression()
lr.fit(X_train, y_train)
Out[12]:
In [13]:
y_pred = lr.predict(X_test)
In [14]:
accuracy_score(y_pred, y_test)
Out[14]:
Grid Search für beste Hyperparameter¶
c: je höher, desto weniger ist das Model regularized (default:1.0)
In [15]:
param_grid = {'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000] }
In [16]:
clf = GridSearchCV(LogisticRegression(penalty='l2'), param_grid)
In [17]:
clf.fit(X_train, y_train)
Out[17]:
In [18]:
clf.best_params_
Out[18]:
In [19]:
# Logistic Regression it c= 0.01
lr_reg = LogisticRegression(C=0.01)
lr_reg.fit(X_train, y_train)
Out[19]:
In [20]:
y_pred_reg = lr_reg.predict(X_test)
In [21]:
y_proba = lr_reg.predict_proba(X_test)
In [22]:
accuracy_score(y_test, y_pred_reg)
Out[22]:
Für die Roc Kurve brauche ich Thresholds, sonst gibt es nur einen Punkt und sie gibt Accuracy aus.
In [23]:
roc_auc_score(y_test, y_pred_reg)
Out[23]:
In [24]:
print(classification_report(y_test, y_pred_reg))
In [25]:
cm = confusion_matrix(y_test, y_pred_reg)
In [26]:
cm
Out[26]:
In [27]:
plt.rcParams['font.size'] = 20
ax= plt.subplot()
sns.heatmap(cm, annot=True, ax = ax, cmap='Set2', fmt=".1f"); #annot=True to annotate cells
# labels, title and ticks
ax.set_xlabel('Predicted labels', fontsize=18);
ax.set_ylabel('True labels', fontsize=18);
ax.set_title('Confusion Matrix');
ax.xaxis.set_ticklabels(['0', '1']);
ax.yaxis.set_ticklabels(['0', '1']);
plt.tight_layout()
Welche Patienten wurden falsch zugeordnet?¶
In [28]:
results = pd.DataFrame({'y_test': y_test, 'y_pred': y_pred_reg, 'y_proba': y_proba[:,1]})
In [29]:
results = results.sort_index()
In [30]:
# alle falsch klassifizierten Patienten
results[(y_test == 1) == (y_pred_reg == 0)]
Out[30]:
In [31]:
results[(y_test == 1) & (y_pred_reg == 0)]
Out[31]:
In [32]:
df.iloc[[42, 101, 139], :]
Out[32]:
In [33]:
# Median der Werte über 0,5
results.loc[results.y_proba > 0.5].median()
Out[33]:
In [34]:
results.loc[results.y_proba > 0.5].min()
Out[34]:
In [35]:
# Es gibt keine Werte bei 0.5, daher ist das kde Plot unten irgendwie irreführend.
In [36]:
figsize(10, 8)
sns.kdeplot(results.y_proba, shade = True)
plt.axvline(0.5, 0, 1, color='red')
Out[36]:
In [37]:
figsize(10, 8)
plt.hist(results.y_proba, bins = 35)
plt.axvline(0.5, 0, 1, color='red')
Out[37]:
In [38]:
# Recall berechnen
tpr = cm[1,1]/(cm[1,1]+cm[1,0])
tpr
Out[38]:
In [39]:
fpr= cm[0,1]/(cm[0,1]+cm[0,0])
fpr
Out[39]:
In [40]:
y_pred_prob = lr_reg.predict_proba(X_test)[:,1]
In [41]:
fpr, tpr, thresholds = roc_curve(y_test, y_pred_prob, drop_intermediate=False)
In [42]:
thresholds
Out[42]:
Es scheint verwunderlich, dass der erste Werte von Thresholds über 1 liegt. Dies ist kein Bug, sondern soll so sein, damit der Wert 0 für TPR und FPR angezeigt werden kann.
In [43]:
figsize(10, 7)
plt.plot(fpr, tpr)
plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.title('ROC Curve Heart Disease')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.grid(True)
In [44]:
fpr
Out[44]:
In [45]:
plt.subplots(figsize=(15, 8))
plt.plot(fpr, tpr, 'o-', label="ROC curve")
plt.plot(np.linspace(0,1,10), np.linspace(0,1,10), label="diagonal")
for x, y, txt in zip(fpr[::5], tpr[::5], thresholds[::5]):
plt.annotate(np.round(txt,2), (x, y-0.04))
rnd_idx = 27
plt.annotate( 'this point refers to the tpr and the fpr\n at a probability threshold of {}'.format(np.round(thresholds[rnd_idx], 2)),
xy=(fpr[rnd_idx], tpr[rnd_idx]), xytext=(fpr[rnd_idx]+0.2, tpr[rnd_idx]-0.25),
arrowprops=dict(facecolor='black', lw=2, arrowstyle='->'),)
plt.legend(loc="upper left")
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
Out[45]:
In [46]:
print(tuple(zip(fpr[::5], tpr[::5], thresholds[::5])))
ROC Curve mit Plotly¶
In [47]:
trace1 = go.Scatter(x=fpr, y=tpr,
mode='lines',
line=dict(color='darkorange'),
name='ROC curve'
)
trace2 = go.Scatter(x=[-0.05, 1.05], y=[-0.05, 1.05],
mode='lines',
line=dict(color='navy', dash='dash'),
showlegend=False)
layout = go.Layout(title='Receiver operating characteristic Heart Disease',
xaxis=dict(title='False Positive Rate'),
yaxis=dict(title='True Positive Rate'))
fig = go.Figure(data=[trace1, trace2], layout=layout)
In [48]:
py.iplot(fig)
Test mit class_weight = 'balanced'¶
In [49]:
lr_bal = LogisticRegression(C=0.01, class_weight='balanced')
lr_bal.fit(X_train, y_train)
Out[49]:
In [50]:
y_pred_bal = lr_bal.predict(X_test)
In [51]:
accuracy_score(y_pred_bal, y_test)
Out[51]:
Logistic Regression mit C = 0.01 hat die beste Accuracy von 0.885¶
Model aus Datensatz mit Polynomial und Interaction Features¶
In [52]:
df_poly = pd.read_csv('df_poly.csv')
df_poly.head()
Out[52]:
In [53]:
x_p = df_poly.drop('target', axis=1)
y = df_poly.target
x_p.shape, y.shape
Out[53]:
In [54]:
X_train_p, X_test_p, y_train, y_test = train_test_split(x_p, y, test_size=0.2, random_state=42)
X_train_p.shape, X_test_p.shape, y_train.shape, y_test.shape
Out[54]:
Standardisieren des Polynomial Datensatzes
In [55]:
sc.fit(X_train_p)
X_train_p =sc.transform(X_train_p);
X_test_p =sc.transform(X_test_p);
In [56]:
lr_reg_p = LogisticRegression(C=0.01)
lr_reg_p.fit(X_train_p, y_train)
Out[56]:
In [57]:
y_p_pred = lr_reg_p.predict(X_test_p)
In [58]:
accuracy_score(y_p_pred, y_test)
Out[58]:
Mit Polynomial Features performt das Model auf dem Test-Set gleich.
Verschiedene Thresholds testen, um optimalen Recall zu finden¶
Bei der Auswahl einer Metrik für die Güte eines Models ist die Problemstellung von großer Bedeutung. Soll vorhergesagt werden, ob ein Patient krank oder gesund ist, ist die Metrik Recall wichtiger als der Accuracy-Score. Der Grund hierfür ist, dass lieber einige gesunde Patienten weiter getestet werden, als das ein kranker Patient als gesund eingestuft wird.
Zudem sollen nun nicht die Klassen direkt vorhergesagt werden, sondern die Wahrscheinlichkeit für die Zugehörigkeit zu einer Klasse. So können Risikogruppen identifiziert werden.
In [59]:
# Dataframe aus den Vorhersagen in Wahrscheinlichkeiten
pred_proba_df = pd.DataFrame(lr_reg.predict_proba(X_test))
# Liste von Entscheidungsgrenzen
threshold_list = [0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.45,0.5,0.55,0.6,0.65,.7,.75,.8,.85,.9,.95,.99]
tpr_list = []
fpr_list = []
for i in threshold_list:
print ('\n****** For threshold = {} ******'.format(i))
y_test_pred = pred_proba_df.applymap(lambda x: 1 if x > i else 0)
# y_test und Spalte für 1 aus y_pred umwandeln, Accuracy Score aller Werte i berechnen
test_accuracy = accuracy_score(y_test.as_matrix().reshape(y_test.as_matrix().size, 1),
y_test_pred.iloc[:,1].as_matrix().reshape(y_test_pred.iloc[:,1].as_matrix().size, 1))
print('Testing Accuracy: {}'.format(test_accuracy))
# Confusion Matrix für alle Werte i
con = confusion_matrix(y_test.as_matrix().reshape(y_test.as_matrix().size, 1),
y_test_pred.iloc[:,1].as_matrix().reshape(y_test_pred.iloc[:,1].as_matrix().size, 1))
tpr = con[1,1]/(con[1,1]+con[1,0])
fpr = con[0,1]/(con[0,1]+con[0,0])
tpr_list.append(tpr)
fpr_list.append(fpr)
print('Confusion Matrix: ')
print(con)
print('True Positve Rate: ', tpr)
print('False Positive Rate: ', fpr)
In [60]:
rev = [i for i in threshold_list[::-1]]
rev
Out[60]:
In [61]:
[i for i in reversed(threshold_list)]
Out[61]:
In [62]:
rates = pd.DataFrame({'Threshold' : threshold_list,
'True Positive Rate': tpr_list,
'False Positive Rate': fpr_list})
In [63]:
rates
Out[63]:
In [64]:
rate = rates.drop([0, 1, 6, 7, 17, 18, 19])
In [65]:
rate = rate.reset_index(drop=True)
In [66]:
rate
Out[66]:
In [67]:
tpr = rate['True Positive Rate']
fpr = rate['False Positive Rate']
th = rate['Threshold']
figsize(15, 15)
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
plt.title("ROC Curve")
plt.scatter(fpr, tpr)
for i, txt in enumerate(th):
plt.annotate(txt, (fpr[i]+0.01, tpr[i]))
In [ ]: