#load packages library(dplyr) library(SDMTools) library(descr) library(ROCR) library(ggplot2) library(boot) library(MASS) library(SDMTools) library(ROCR) #read in data defaultData <- read.csv("https://assets.datacamp.com/production/repositories/1861/datasets/0b0772985a8676c3613e8ac2c6053f5e60a3aebd/defaultData.csv", sep = ";") defaultData <- defaultData %>% mutate_at(c("sex","marriage", "PaymentDefault", "education"), factor ) #for comparison: silently calculate in-sample values: #part 1: #split data in train and test set set.seed(534381) defaultData$isTrain <- rbinom(nrow(defaultData), 1, 0.66) train <- subset(defaultData, defaultData$isTrain == 1) test <- subset(defaultData, defaultData$isTrain == 0) # Modeling logitTrainFull <- glm(PaymentDefault ~ limitBal + sex + education + marriage + age + pay1 + pay2 + pay3 + pay5 + billAmt1 + billAmt2 + billAmt5 + payAmt1 + payAmt2 + payAmt3 + payAmt4 + payAmt5 + payAmt6, family = binomial, data = train) logitTrainNew <- glm(formula = PaymentDefault ~ sex + education + marriage + pay1 + pay2 + pay3 + pay5 + billAmt1 + billAmt2 + payAmt1 + payAmt2 + payAmt6, family = binomial, data = train) logitModelFull <- glm(PaymentDefault ~ limitBal + sex + education + marriage + age + pay1 + pay2 + pay3 + pay4 + pay5 + pay6 + billAmt1 + billAmt2 + billAmt3 + billAmt4 + billAmt5 + billAmt6 + payAmt1 + payAmt2 + payAmt3 + payAmt4 + payAmt5 + payAmt6, family = binomial, defaultData) logitModelNew <- glm(PaymentDefault ~ marriage + pay1 + pay3 + pay5 + billAmt1 + billAmt6 + payAmt1 + payAmt2 + payAmt5 + payAmt6, family = binomial, defaultData) #in sample accuracy for comparison just on train data #not shown for students: # Prediction on train dataset train$predNew <- predict(logitTrainNew, type = "response", na.action = na.exclude) # In-sample confusion matrix confMatrixNew <- confusion.matrix(train$PaymentDefault, train$predNew, threshold = 0.3) #logitModelNew accuracyNewInSample <- sum(diag(confMatrixNew)) / sum(confMatrixNew) train$predFull <- predict(logitTrainFull, type = "response", na.action = na.exclude) # In-sample confusion matrix confMatrixFull <- confusion.matrix(train$PaymentDefault, train$predFull, threshold = 0.3) #logitModelNew accuracyFullInSample <- sum(diag(confMatrixFull)) / sum(confMatrixFull) # Do the same for logitModelFull logitTrainFull <- glm(PaymentDefault ~ limitBal + sex + education + marriage + age + pay1 + pay2 + pay3 + pay4 + pay5 + pay6 + billAmt1 + billAmt2 + billAmt3 + billAmt4 + billAmt5 + billAmt6 + payAmt1 + payAmt2 + payAmt3 + payAmt4 + payAmt5 + payAmt6, family = binomial, data = train) # Out-of-sample prediction test$predFull <- predict(logitTrainFull, type = "response", newdata = test) # out-of-sample accuracy confMatrixModelFull <- confusion.matrix(test$PaymentDefault, test$predFull, threshold = 0.3) confMatrixModelFull accuracyFull <- sum(diag(confMatrixModelFull)) / sum(confMatrixModelFull) accuracyFull formulaLogit <- as.formula(PaymentDefault ~ limitBal + sex + education + marriage + age + pay1 + pay2 + pay3 + pay5 + billAmt1 + billAmt2 + billAmt5 + payAmt1 + payAmt2 + payAmt3 + payAmt4 + payAmt5 + payAmt6)

Danger of overfitting

Exercice interactif pratique