bank

Load Data and Packages

library(tidyverse)

## ── Attaching packages ─────────────────────────────────────── tidyverse 1.3.2 ──
## ✔ ggplot2 3.3.6     ✔ purrr   0.3.4
## ✔ tibble  3.2.1     ✔ dplyr   1.1.1
## ✔ tidyr   1.2.0     ✔ stringr 1.4.0
## ✔ readr   2.1.2     ✔ forcats 0.5.1
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag()    masks stats::lag()

library(GGally)

## Registered S3 method overwritten by 'GGally':
##   method from   
##   +.gg   ggplot2

library(skimr)
library(reshape2)

## 
## Attaching package: 'reshape2'
## 
## The following object is masked from 'package:tidyr':
## 
##     smiths

library(caret)

## Loading required package: lattice
## 
## Attaching package: 'caret'
## 
## The following object is masked from 'package:purrr':
## 
##     lift

library(olsrr)

## 
## Attaching package: 'olsrr'
## 
## The following object is masked from 'package:datasets':
## 
##     rivers

library(car)

## Loading required package: carData
## 
## Attaching package: 'car'
## 
## The following object is masked from 'package:dplyr':
## 
##     recode
## 
## The following object is masked from 'package:purrr':
## 
##     some

library(glmnet)

## Loading required package: Matrix
## 
## Attaching package: 'Matrix'
## 
## The following objects are masked from 'package:tidyr':
## 
##     expand, pack, unpack
## 
## Loaded glmnet 4.1-6

library(pROC)

## Type 'citation("pROC")' for a citation.
## 
## Attaching package: 'pROC'
## 
## The following objects are masked from 'package:stats':
## 
##     cov, smooth, var

library(ResourceSelection)

## ResourceSelection 0.3-5   2019-07-22

library(MASS)

## 
## Attaching package: 'MASS'
## 
## The following object is masked from 'package:olsrr':
## 
##     cement
## 
## The following object is masked from 'package:dplyr':
## 
##     select

library(class)
#library(gtsummary)

bank = read.csv("bank-full.csv", stringsAsFactors = T)
head(bank)

##   age          job marital education default balance housing loan contact day
## 1  58   management married  tertiary      no    2143     yes   no unknown   5
## 2  44   technician  single secondary      no      29     yes   no unknown   5
## 3  33 entrepreneur married secondary      no       2     yes  yes unknown   5
## 4  47  blue-collar married   unknown      no    1506     yes   no unknown   5
## 5  33      unknown  single   unknown      no       1      no   no unknown   5
## 6  35   management married  tertiary      no     231     yes   no unknown   5
##   month duration campaign pdays previous poutcome  y
## 1   may      261        1    -1        0  unknown no
## 2   may      151        1    -1        0  unknown no
## 3   may       76        1    -1        0  unknown no
## 4   may       92        1    -1        0  unknown no
## 5   may      198        1    -1        0  unknown no
## 6   may      139        1    -1        0  unknown no

summary(bank)

##       age                 job           marital          education    
##  Min.   :18.00   blue-collar:9732   divorced: 5207   primary  : 6851  
##  1st Qu.:33.00   management :9458   married :27214   secondary:23202  
##  Median :39.00   technician :7597   single  :12790   tertiary :13301  
##  Mean   :40.94   admin.     :5171                    unknown  : 1857  
##  3rd Qu.:48.00   services   :4154                                     
##  Max.   :95.00   retired    :2264                                     
##                  (Other)    :6835                                     
##  default        balance       housing      loan            contact     
##  no :44396   Min.   : -8019   no :20081   no :37967   cellular :29285  
##  yes:  815   1st Qu.:    72   yes:25130   yes: 7244   telephone: 2906  
##              Median :   448                           unknown  :13020  
##              Mean   :  1362                                            
##              3rd Qu.:  1428                                            
##              Max.   :102127                                            
##                                                                        
##       day            month          duration         campaign     
##  Min.   : 1.00   may    :13766   Min.   :   0.0   Min.   : 1.000  
##  1st Qu.: 8.00   jul    : 6895   1st Qu.: 103.0   1st Qu.: 1.000  
##  Median :16.00   aug    : 6247   Median : 180.0   Median : 2.000  
##  Mean   :15.81   jun    : 5341   Mean   : 258.2   Mean   : 2.764  
##  3rd Qu.:21.00   nov    : 3970   3rd Qu.: 319.0   3rd Qu.: 3.000  
##  Max.   :31.00   apr    : 2932   Max.   :4918.0   Max.   :63.000  
##                  (Other): 6060                                    
##      pdays          previous           poutcome       y        
##  Min.   : -1.0   Min.   :  0.0000   failure: 4901   no :39922  
##  1st Qu.: -1.0   1st Qu.:  0.0000   other  : 1840   yes: 5289  
##  Median : -1.0   Median :  0.0000   success: 1511              
##  Mean   : 40.2   Mean   :  0.5803   unknown:36959              
##  3rd Qu.: -1.0   3rd Qu.:  0.0000                              
##  Max.   :871.0   Max.   :275.0000                              
##

There are a total of 45211 rows and 17 columns.

# Look at structure
str(bank)

## 'data.frame':    45211 obs. of  17 variables:
##  $ age      : int  58 44 33 47 33 35 28 42 58 43 ...
##  $ job      : Factor w/ 12 levels "admin.","blue-collar",..: 5 10 3 2 12 5 5 3 6 10 ...
##  $ marital  : Factor w/ 3 levels "divorced","married",..: 2 3 2 2 3 2 3 1 2 3 ...
##  $ education: Factor w/ 4 levels "primary","secondary",..: 3 2 2 4 4 3 3 3 1 2 ...
##  $ default  : Factor w/ 2 levels "no","yes": 1 1 1 1 1 1 1 2 1 1 ...
##  $ balance  : int  2143 29 2 1506 1 231 447 2 121 593 ...
##  $ housing  : Factor w/ 2 levels "no","yes": 2 2 2 2 1 2 2 2 2 2 ...
##  $ loan     : Factor w/ 2 levels "no","yes": 1 1 2 1 1 1 2 1 1 1 ...
##  $ contact  : Factor w/ 3 levels "cellular","telephone",..: 3 3 3 3 3 3 3 3 3 3 ...
##  $ day      : int  5 5 5 5 5 5 5 5 5 5 ...
##  $ month    : Factor w/ 12 levels "apr","aug","dec",..: 9 9 9 9 9 9 9 9 9 9 ...
##  $ duration : int  261 151 76 92 198 139 217 380 50 55 ...
##  $ campaign : int  1 1 1 1 1 1 1 1 1 1 ...
##  $ pdays    : int  -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 ...
##  $ previous : int  0 0 0 0 0 0 0 0 0 0 ...
##  $ poutcome : Factor w/ 4 levels "failure","other",..: 4 4 4 4 4 4 4 4 4 4 ...
##  $ y        : Factor w/ 2 levels "no","yes": 1 1 1 1 1 1 1 1 1 1 ...

# Skim the dataset
skim(bank)

Data summary
Name	bank
Number of rows	45211
Number of columns	17
_______________________
Column type frequency:
factor	10
numeric	7
________________________
Group variables	None

Variable type: factor

skim_variable	complete_rate	ordered	n_unique	top_counts
job	1	FALSE	12	blu: 9732, man: 9458, tec: 7597, adm: 5171
marital	1	FALSE	3	mar: 27214, sin: 12790, div: 5207
education	1	FALSE	4	sec: 23202, ter: 13301, pri: 6851, unk: 1857
default	1	FALSE	2	no: 44396, yes: 815
housing	1	FALSE	2	yes: 25130, no: 20081
loan	1	FALSE	2	no: 37967, yes: 7244
contact	1	FALSE	3	cel: 29285, unk: 13020, tel: 2906
month	1	FALSE	12	may: 13766, jul: 6895, aug: 6247, jun: 5341
poutcome	1	FALSE	4	unk: 36959, fai: 4901, oth: 1840, suc: 1511
y	1	FALSE	2	no: 39922, yes: 5289

Variable type: numeric

skim_variable	complete_rate	mean	sd	p0	p25	p50	p75	p100	hist
age	1	40.94	10.62	18	33	39	48	95	▅▇▃▁▁
balance	1	1362.27	3044.77	-8019	72	448	1428	102127	▇▁▁▁▁
day	1	15.81	8.32	1	8	16	21	31	▇▆▇▆▆
duration	1	258.16	257.53	0	103	180	319	4918	▇▁▁▁▁
campaign	1	2.76	3.10	1	1	2	3	63	▇▁▁▁▁
pdays	1	40.20	100.13	-1	-1	-1	-1	871	▇▁▁▁▁
previous	1	0.58	2.30	0	0	0	0	275	▇▁▁▁▁

Check for missing values

No null data in our dataset.

colSums(is.na(bank))

##       age       job   marital education   default   balance   housing      loan 
##         0         0         0         0         0         0         0         0 
##   contact       day     month  duration  campaign     pdays  previous  poutcome 
##         0         0         0         0         0         0         0         0 
##         y 
##         0

sum(is.na(bank))

## [1] 0

Objective 1

Exploratory Data Analysis(EDA)

Looking at the subscription count, we can see that we have heavily imbalanced dataset.There are almost 40k Nos while there is about about 5k Yes’s.

There are several plots that are noteworthy. One is the housing barplot. Even though, we see that the number of count for those with no housing is lower, the subscription count is higher.

# yes or no subscruption count
ggplot(bank, aes(x = y)) +
  geom_bar()

# Job Count
ggplot(bank, aes(x = job, fill = y)) +
  geom_bar() +
  theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1)) + ggtitle("Subscriptions vs. Job Count")

# Education Count
ggplot(bank, aes(x = education, fill = y)) +
  geom_bar() +
  theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1)) + ggtitle("Subscriptions vs. Education Count")

# Age Distribution
ggplot(bank, aes(x = age, fill = y)) +
  geom_histogram(bins = 50) + ggtitle("Subscriptions vs. Age Distribution")

# Duration Distribution
ggplot(bank, aes(x = duration, fill = y)) +
  geom_histogram(bins = 50) + ggtitle("Subscriptions vs. Duration Distribution")

ggplot(bank, aes(x = log(duration), fill = y)) +
  geom_histogram(bins = 50) + ggtitle("Subscriptions vs. log(Duration) Distribution")

## Warning: Removed 3 rows containing non-finite values (stat_bin).

# Balance Distribution
ggplot(bank, aes(x = balance, fill = y)) +
  geom_histogram(bins = 20) + ggtitle("Subscriptions vs. Balance Distribution")

# Housing Count
ggplot(bank, aes(x = housing, fill = y)) +
  geom_bar() + ggtitle("Subscriptions vs. Housing Count")

# Loan 
ggplot(data = bank, aes(x = loan , fill = y)) + 
  geom_bar(stat = 'count', position = 'dodge') + 
  xlab("Loan")+ylab("Response") + ggtitle("Subscriptions vs. Loans")

# Marital Status 
ggplot(data = bank, aes(x = marital, fill = y)) + 
  geom_bar(stat = 'count', position = 'dodge') + 
  xlab("Marital")+ylab("Response") + ggtitle("Subscriptions vs. Marital Status")

# Default
ggplot(data = bank, aes(x = default ,  fill = y)) + 
  geom_bar(stat = 'count', position = 'dodge') + 
  xlab("Default")+ylab("Response") + ggtitle("Subscriptions vs. Default")

# Poutcome
ggplot(data = bank, aes(x = poutcome ,  fill = y)) + 
  geom_bar(stat = 'count', position = 'dodge') + 
  xlab("Poutcome")+ylab("Response") + ggtitle("Subscriptions vs. Previous Outcome")

# Previous 
ggplot(data = bank, aes(x = previous, fill = y)) +
  geom_histogram() +
  labs(title = "Distribution of Previous Campaign", 
       x = "Previous Campaign", y = "Count")

library(dplyr)
# group jobs by percent 
g1<-bank %>% 
  group_by(job,y) %>%
  summarise(cnt=n()) %>%
  mutate(perc=round(cnt/sum(cnt),4))%>%
  arrange(desc(perc))

## `summarise()` has grouped output by 'job'. You can override using the `.groups`
## argument.

# Both Yes & No
# ggplot(g1,aes(x=reorder(job,-perc),y=perc,colour=y))+
#   geom_bar(aes(fill=y),show.legend=T,stat="identity") + ylab("Proportion of Subscriptions") + xlab("Job Type") + ggtitle("Propotion Subscriptions vs. Job Type")

ggplot(g1[13:24,],aes(x=reorder(job,-perc),y=perc,colour=job))+
  geom_bar(aes(fill=job),show.legend=T,stat="identity") + ylab("Proportion of Subscriptions") + xlab("Job Type") + ggtitle("Propotion Subscriptions vs. Job Type")

# group poutcome by percent
g2<-bank %>% 
  group_by(poutcome,y) %>%
  summarise(cnt=n()) %>%
  mutate(perc=round(cnt/sum(cnt),4))%>%
  arrange(desc(perc))

## `summarise()` has grouped output by 'poutcome'. You can override using the
## `.groups` argument.

# ggplot(g2,aes(x=reorder(poutcome,-perc),y=perc,colour=y))+
#   geom_bar(aes(fill=y),show.legend=T,stat="identity")  + ylab("Proportion of Subscriptions") + xlab("Previous Outcome") + ggtitle(" Proportion of Subscriptions vs. Previous Outcome")

ggplot(g2[c(4, 6, 7, 8),],aes(x=reorder(poutcome,-perc),y=perc,colour=poutcome))+
  geom_bar(aes(fill=poutcome),show.legend=T,stat="identity")  + ylab("Proportion of Subscriptions") + xlab("Previous Outcome") + ggtitle(" Proportion of Subscriptions vs. Previous Outcome")

# group default by percent 
g3<-bank %>% 
  group_by(default,y) %>%
  summarise(cnt=n()) %>%
  mutate(perc=round(cnt/sum(cnt),4))%>%
  arrange(desc(perc))

## `summarise()` has grouped output by 'default'. You can override using the
## `.groups` argument.

ggplot(g3,aes(x=reorder(default,-perc),y=perc,colour=y))+
  geom_bar(aes(fill=y),show.legend=T,stat="identity")  + ylab("Proportion of Subscriptions") + xlab("Default") + ggtitle(" Proportion of Subscriptions vs. Default")

# group month by proportion 
g4<-bank %>% 
  group_by(month,y) %>%
  summarise(cnt=n()) %>%
  mutate(perc=round(cnt/sum(cnt),4))%>%
  arrange(desc(perc))

## `summarise()` has grouped output by 'month'. You can override using the
## `.groups` argument.

# ggplot(g4,aes(x=reorder(month,-perc),y=perc,colour=y))+
#   geom_bar(aes(fill=y),show.legend=T,stat="identity")  + ylab("Proportion of Subscriptions") + xlab("Month") + ggtitle(" Proportion of Subscriptions vs. Month")
ggplot(g4[c(12,14:24),],aes(x=reorder(month,-perc),y=perc,colour=month))+
  geom_bar(aes(fill=month),show.legend=T,stat="identity")  + ylab("Proportion of Subscriptions") + xlab("Month") + ggtitle(" Proportion of Subscriptions vs. Month")

# group education by proportion 
g5 <-bank %>% 
  group_by(education,y) %>%
  summarise(cnt=n()) %>%
  mutate(perc=round(cnt/sum(cnt),4))%>%
  arrange(desc(perc))

## `summarise()` has grouped output by 'education'. You can override using the
## `.groups` argument.

g5

## # A tibble: 8 × 4
## # Groups:   education [4]
##   education y       cnt   perc
##   <fct>     <fct> <int>  <dbl>
## 1 primary   no     6260 0.914 
## 2 secondary no    20752 0.894 
## 3 unknown   no     1605 0.864 
## 4 tertiary  no    11305 0.850 
## 5 tertiary  yes    1996 0.150 
## 6 unknown   yes     252 0.136 
## 7 secondary yes    2450 0.106 
## 8 primary   yes     591 0.0863

ggplot(g5[c(5:8),],aes(x=reorder(education,-perc),y=perc,colour=education))+
  geom_bar(aes(fill=education),show.legend=T,stat="identity")  + ylab("Proportion of Subscriptions") + xlab("Month") + ggtitle(" Proportion of Subscriptions vs. Month")

# group marital status by proportion 
g6 <-bank %>% 
  group_by(marital,y) %>%
  summarise(cnt=n()) %>%
  mutate(perc=round(cnt/sum(cnt),4))%>%
  arrange(desc(perc))

## `summarise()` has grouped output by 'marital'. You can override using the
## `.groups` argument.

g6

## # A tibble: 6 × 4
## # Groups:   marital [3]
##   marital  y       cnt  perc
##   <fct>    <fct> <int> <dbl>
## 1 married  no    24459 0.899
## 2 divorced no     4585 0.880
## 3 single   no    10878 0.850
## 4 single   yes    1912 0.150
## 5 divorced yes     622 0.120
## 6 married  yes    2755 0.101

ggplot(g6[c(4:6),],aes(x=reorder(marital,-perc),y=perc,colour=marital))+
  geom_bar(aes(fill=marital),show.legend=T,stat="identity")  + ylab("Proportion of Subscriptions") + xlab("Month") + ggtitle(" Proportion of Subscriptions vs. Marital")

Correlation Matrix

After creating a dummy column to make our y column numeric, we can see that there is hardly any correlation between our response variable and the other variables. The strongest relationship is y_num and duration at 0.395.

# make new numeric column for "y" column
bank$y_num = num(ifelse(bank$y == "yes",1,0))

## Correlation map
num_cols = unlist(lapply(bank, is.numeric))
data_num = bank[ , num_cols] 
corr_mat = round(cor(data_num),2)
 
# reduce the size of correlation matrix
melted_corr_mat = melt(corr_mat)
# head(melted_corr_mat)
 
# plotting the correlation heatmap
ggplot(data = melted_corr_mat, aes(x=Var1, y=Var2,
                                   fill=value)) +
geom_tile() +
geom_text(aes(Var2, Var1, label = value),
          color = "white", size = 4) +
  ggtitle("Heatmap of Correalation For Numeric Variables")

# ggpairs
ggpairs(data_num)

LOESS

# Age (check)
ggplot(bank,aes(x=age,y=y_num))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE) + ggtitle("Age vs. Subscription Loess Plot")

## `geom_smooth()` using formula 'y ~ x'

# Balance
ggplot(bank,aes(x=balance,y=y_num))+geom_point()+
  geom_smooth(method="loess",size=1,span=0.1, se = FALSE) + ggtitle("Balance vs. Subscription Loess Plot")

## `geom_smooth()` using formula 'y ~ x'

# Duration (maybe add poly to it)
ggplot(bank,aes(x=duration,y=y_num))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE) + ggtitle("Duration vs. Subscription Loess Plot")

## `geom_smooth()` using formula 'y ~ x'

# Pdays (check)
ggplot(bank,aes(x=pdays,y=y_num))+geom_point()+
  geom_smooth(method="loess",size=1,span=1, se = FALSE) + ggtitle("Pdays vs. Subscription Loess Plot")

## `geom_smooth()` using formula 'y ~ x'

# campaign 
ggplot(bank,aes(x=campaign,y=y_num))+geom_point()+
  geom_smooth(method="loess",size=1,span=1, se = FALSE) + ggtitle("Campain vs. Subscription Loess Plot")

## `geom_smooth()` using formula 'y ~ x'

# day
ggplot(bank,aes(x=day,y=y_num))+geom_point()+
  geom_smooth(method="loess",size=1,span=1, se = FALSE) + ggtitle("Day vs. Subscription Loess Plot")

## `geom_smooth()` using formula 'y ~ x'

Base Model

AUROC: 0.9085 LogLoss: 0.2409315 AIC: 15206 Accuracy: 0.814 Sensitivity: 0.8064 Specificity: 0.8714

Note that because we have such unbalanced data, our accuracy is always expected to be high. Only 10% of the results were “yes”, so just picking “no” 100% of the time would result in an accuracy of 90%.

However, once we try and increase the specificity, we realize that the ideal threshold for this base model is 0.096, which reduces our accuracy down to 81%. The decrease in accuracy results in a much better improvement and equality in sensitivity and specificity.

bank = bank[,-c(18)] # remove y_num from dataset

set.seed(124)
trainIndex = createDataPartition(bank$y, p= .7, list = F) 
training = bank[trainIndex,]
test = bank[-trainIndex,]

model1 = glm(factor(y)~., data = training, family = "binomial")
summary(model1)

## 
## Call:
## glm(formula = factor(y) ~ ., family = "binomial", data = training)
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -5.7184  -0.3780  -0.2526  -0.1485   3.4431  
## 
## Coefficients:
##                      Estimate Std. Error z value Pr(>|z|)    
## (Intercept)        -2.540e+00  2.202e-01 -11.537  < 2e-16 ***
## age                 1.136e-03  2.645e-03   0.429 0.667657    
## jobblue-collar     -3.342e-01  8.792e-02  -3.801 0.000144 ***
## jobentrepreneur    -3.327e-01  1.507e-01  -2.207 0.027298 *  
## jobhousemaid       -5.736e-01  1.674e-01  -3.427 0.000610 ***
## jobmanagement      -1.343e-01  8.788e-02  -1.528 0.126495    
## jobretired          2.826e-01  1.154e-01   2.449 0.014308 *  
## jobself-employed   -1.625e-01  1.308e-01  -1.242 0.214069    
## jobservices        -2.555e-01  1.024e-01  -2.496 0.012575 *  
## jobstudent          4.549e-01  1.280e-01   3.553 0.000381 ***
## jobtechnician      -1.335e-01  8.258e-02  -1.616 0.106106    
## jobunemployed      -1.438e-01  1.323e-01  -1.087 0.276846    
## jobunknown         -1.199e-01  2.639e-01  -0.454 0.649696    
## maritalmarried     -1.740e-01  7.040e-02  -2.472 0.013454 *  
## maritalsingle       1.203e-01  8.053e-02   1.494 0.135171    
## educationsecondary  1.879e-01  7.791e-02   2.411 0.015896 *  
## educationtertiary   3.713e-01  9.000e-02   4.125 3.70e-05 ***
## educationunknown    1.147e-01  1.259e-01   0.911 0.362262    
## defaultyes          1.401e-01  1.800e-01   0.778 0.436585    
## balance             1.610e-05  5.839e-06   2.758 0.005809 ** 
## housingyes         -7.378e-01  5.261e-02 -14.024  < 2e-16 ***
## loanyes            -4.688e-01  7.264e-02  -6.454 1.09e-10 ***
## contacttelephone   -1.198e-01  8.894e-02  -1.346 0.178159    
## contactunknown     -1.566e+00  8.780e-02 -17.833  < 2e-16 ***
## day                 7.758e-03  2.984e-03   2.600 0.009323 ** 
## monthaug           -7.622e-01  9.386e-02  -8.120 4.65e-16 ***
## monthdec            6.109e-01  2.070e-01   2.951 0.003164 ** 
## monthfeb           -2.235e-01  1.065e-01  -2.100 0.035764 *  
## monthjan           -1.221e+00  1.423e-01  -8.578  < 2e-16 ***
## monthjul           -8.678e-01  9.274e-02  -9.357  < 2e-16 ***
## monthjun            3.016e-01  1.127e-01   2.676 0.007442 ** 
## monthmar            1.437e+00  1.467e-01   9.790  < 2e-16 ***
## monthmay           -4.477e-01  8.626e-02  -5.190 2.10e-07 ***
## monthnov           -8.698e-01  1.003e-01  -8.676  < 2e-16 ***
## monthoct            8.064e-01  1.285e-01   6.274 3.51e-10 ***
## monthsep            6.953e-01  1.419e-01   4.902 9.50e-07 ***
## duration            4.156e-03  7.794e-05  53.314  < 2e-16 ***
## campaign           -9.286e-02  1.213e-02  -7.658 1.89e-14 ***
## pdays               2.810e-04  3.650e-04   0.770 0.441484    
## previous            1.187e-02  7.506e-03   1.581 0.113860    
## poutcomeother       2.454e-01  1.057e-01   2.321 0.020287 *  
## poutcomesuccess     2.315e+00  9.842e-02  23.520  < 2e-16 ***
## poutcomeunknown    -4.092e-02  1.119e-01  -0.366 0.714603    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 22845  on 31648  degrees of freedom
## Residual deviance: 15120  on 31606  degrees of freedom
## AIC: 15206
## 
## Number of Fisher Scoring iterations: 6

vif(model1)

##               GVIF Df GVIF^(1/(2*Df))
## age       2.216913  1        1.488930
## job       4.243808 11        1.067909
## marital   1.461161  2        1.099448
## education 2.225993  3        1.142670
## default   1.020532  1        1.010214
## balance   1.043881  1        1.021705
## housing   1.433865  1        1.197441
## loan      1.067951  1        1.033417
## contact   1.886652  2        1.171987
## day       1.344743  1        1.159631
## month     3.713619 11        1.061451
## duration  1.122924  1        1.059681
## campaign  1.105656  1        1.051502
## pdays     3.756409  1        1.938146
## previous  1.270182  1        1.127024
## poutcome  4.313674  3        1.275874

fitControl = trainControl(method = "repeatedcv", number = 5, repeats = 1, classProbs = TRUE, summaryFunction = mnLogLoss)
logit_base = train(factor(y) ~ ., data = training, method = "glm", trControl = fitControl, metric = "logLoss")
summary(logit_base$finalModel)

## 
## Call:
## NULL
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -5.7184  -0.3780  -0.2526  -0.1485   3.4431  
## 
## Coefficients:
##                      Estimate Std. Error z value Pr(>|z|)    
## (Intercept)        -2.540e+00  2.202e-01 -11.537  < 2e-16 ***
## age                 1.136e-03  2.645e-03   0.429 0.667657    
## `jobblue-collar`   -3.342e-01  8.792e-02  -3.801 0.000144 ***
## jobentrepreneur    -3.327e-01  1.507e-01  -2.207 0.027298 *  
## jobhousemaid       -5.736e-01  1.674e-01  -3.427 0.000610 ***
## jobmanagement      -1.343e-01  8.788e-02  -1.528 0.126495    
## jobretired          2.826e-01  1.154e-01   2.449 0.014308 *  
## `jobself-employed` -1.625e-01  1.308e-01  -1.242 0.214069    
## jobservices        -2.555e-01  1.024e-01  -2.496 0.012575 *  
## jobstudent          4.549e-01  1.280e-01   3.553 0.000381 ***
## jobtechnician      -1.335e-01  8.258e-02  -1.616 0.106106    
## jobunemployed      -1.438e-01  1.323e-01  -1.087 0.276846    
## jobunknown         -1.199e-01  2.639e-01  -0.454 0.649696    
## maritalmarried     -1.740e-01  7.040e-02  -2.472 0.013454 *  
## maritalsingle       1.203e-01  8.053e-02   1.494 0.135171    
## educationsecondary  1.879e-01  7.791e-02   2.411 0.015896 *  
## educationtertiary   3.713e-01  9.000e-02   4.125 3.70e-05 ***
## educationunknown    1.147e-01  1.259e-01   0.911 0.362262    
## defaultyes          1.401e-01  1.800e-01   0.778 0.436585    
## balance             1.610e-05  5.839e-06   2.758 0.005809 ** 
## housingyes         -7.378e-01  5.261e-02 -14.024  < 2e-16 ***
## loanyes            -4.688e-01  7.264e-02  -6.454 1.09e-10 ***
## contacttelephone   -1.198e-01  8.894e-02  -1.346 0.178159    
## contactunknown     -1.566e+00  8.780e-02 -17.833  < 2e-16 ***
## day                 7.758e-03  2.984e-03   2.600 0.009323 ** 
## monthaug           -7.622e-01  9.386e-02  -8.120 4.65e-16 ***
## monthdec            6.109e-01  2.070e-01   2.951 0.003164 ** 
## monthfeb           -2.235e-01  1.065e-01  -2.100 0.035764 *  
## monthjan           -1.221e+00  1.423e-01  -8.578  < 2e-16 ***
## monthjul           -8.678e-01  9.274e-02  -9.357  < 2e-16 ***
## monthjun            3.016e-01  1.127e-01   2.676 0.007442 ** 
## monthmar            1.437e+00  1.467e-01   9.790  < 2e-16 ***
## monthmay           -4.477e-01  8.626e-02  -5.190 2.10e-07 ***
## monthnov           -8.698e-01  1.003e-01  -8.676  < 2e-16 ***
## monthoct            8.064e-01  1.285e-01   6.274 3.51e-10 ***
## monthsep            6.953e-01  1.419e-01   4.902 9.50e-07 ***
## duration            4.156e-03  7.794e-05  53.314  < 2e-16 ***
## campaign           -9.286e-02  1.213e-02  -7.658 1.89e-14 ***
## pdays               2.810e-04  3.650e-04   0.770 0.441484    
## previous            1.187e-02  7.506e-03   1.581 0.113860    
## poutcomeother       2.454e-01  1.057e-01   2.321 0.020287 *  
## poutcomesuccess     2.315e+00  9.842e-02  23.520  < 2e-16 ***
## poutcomeunknown    -4.092e-02  1.119e-01  -0.366 0.714603    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 22845  on 31648  degrees of freedom
## Residual deviance: 15120  on 31606  degrees of freedom
## AIC: 15206
## 
## Number of Fisher Scoring iterations: 6

length(predictors(logit_base))

## [1] 42

# Generate an ROC curve to help us determine the best threshold
logit_base_predictions = predict(logit_base, test, type = "prob")$yes
logit_base_roc = roc(response = test$y, predictor = logit_base_predictions, levels = c("no", "yes"))

## Setting direction: controls < cases

auc(logit_base_roc)

## Area under the curve: 0.9085

plot(logit_base_roc, print.thres = "best", col = "black")

# Predict
threshold = 0.096
logit_base_predictions_categorical = factor(ifelse(predict(logit_base, test, type = "prob")$yes >= threshold, "yes", "no"), levels = c("no", "yes"))
confusionMatrix(data = logit_base_predictions_categorical, reference = factor(test$y, levels = c("no", "yes")))

## Confusion Matrix and Statistics
## 
##           Reference
## Prediction   no  yes
##        no  9658  204
##        yes 2318 1382
##                                           
##                Accuracy : 0.814           
##                  95% CI : (0.8074, 0.8206)
##     No Information Rate : 0.8831          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0.4295          
##                                           
##  Mcnemar's Test P-Value : <2e-16          
##                                           
##             Sensitivity : 0.8064          
##             Specificity : 0.8714          
##          Pos Pred Value : 0.9793          
##          Neg Pred Value : 0.3735          
##              Prevalence : 0.8831          
##          Detection Rate : 0.7121          
##    Detection Prevalence : 0.7272          
##       Balanced Accuracy : 0.8389          
##                                           
##        'Positive' Class : no              
##

Adjusted Base Model(GLM-Net)

We remove the insignificant variables from summary(model1).

AUROC: 0.9087 AIC: 15203 Accuracy: 0.8082 Sensitivity: 0.7987 Specificity: 0.8802

However, once we try and increase the specificity, we realize that the ideal threshold for this base model is 0.093, which reduces our accuracy down to 80.82% which is lower than our base model. We see that our sensitivity suffered heavily but our specificity increased slightly.

training1 = training[,-c(1,5,14,15)]
adj.model1 = glm(factor(y)~., data = training1, family = "binomial")
summary(adj.model1)

## 
## Call:
## glm(formula = factor(y) ~ ., family = "binomial", data = training1)
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -5.7147  -0.3782  -0.2526  -0.1488   3.4359  
## 
## Coefficients:
##                      Estimate Std. Error z value Pr(>|z|)    
## (Intercept)        -2.384e+00  1.548e-01 -15.402  < 2e-16 ***
## jobblue-collar     -3.350e-01  8.784e-02  -3.814 0.000137 ***
## jobentrepreneur    -3.277e-01  1.504e-01  -2.178 0.029376 *  
## jobhousemaid       -5.708e-01  1.668e-01  -3.422 0.000622 ***
## jobmanagement      -1.302e-01  8.777e-02  -1.483 0.138063    
## jobretired          3.013e-01  1.037e-01   2.905 0.003669 ** 
## jobself-employed   -1.618e-01  1.308e-01  -1.237 0.216015    
## jobservices        -2.574e-01  1.023e-01  -2.517 0.011850 *  
## jobstudent          4.419e-01  1.255e-01   3.521 0.000430 ***
## jobtechnician      -1.328e-01  8.255e-02  -1.609 0.107566    
## jobunemployed      -1.413e-01  1.322e-01  -1.069 0.284997    
## jobunknown         -1.151e-01  2.637e-01  -0.437 0.662452    
## maritalmarried     -1.780e-01  7.008e-02  -2.539 0.011103 *  
## maritalsingle       1.076e-01  7.548e-02   1.426 0.153845    
## educationsecondary  1.838e-01  7.758e-02   2.369 0.017832 *  
## educationtertiary   3.639e-01  8.938e-02   4.072 4.66e-05 ***
## educationunknown    1.135e-01  1.259e-01   0.902 0.367199    
## balance             1.606e-05  5.818e-06   2.761 0.005768 ** 
## housingyes         -7.357e-01  5.213e-02 -14.113  < 2e-16 ***
## loanyes            -4.653e-01  7.243e-02  -6.425 1.32e-10 ***
## contacttelephone   -1.132e-01  8.784e-02  -1.289 0.197366    
## contactunknown     -1.565e+00  8.770e-02 -17.847  < 2e-16 ***
## day                 7.628e-03  2.982e-03   2.558 0.010528 *  
## monthaug           -7.601e-01  9.374e-02  -8.109 5.12e-16 ***
## monthdec            6.124e-01  2.067e-01   2.962 0.003052 ** 
## monthfeb           -2.227e-01  1.063e-01  -2.096 0.036071 *  
## monthjan           -1.218e+00  1.421e-01  -8.571  < 2e-16 ***
## monthjul           -8.645e-01  9.265e-02  -9.332  < 2e-16 ***
## monthjun            3.015e-01  1.126e-01   2.677 0.007423 ** 
## monthmar            1.436e+00  1.466e-01   9.795  < 2e-16 ***
## monthmay           -4.449e-01  8.621e-02  -5.160 2.47e-07 ***
## monthnov           -8.733e-01  9.966e-02  -8.763  < 2e-16 ***
## monthoct            8.083e-01  1.284e-01   6.296 3.06e-10 ***
## monthsep            6.984e-01  1.418e-01   4.924 8.49e-07 ***
## duration            4.155e-03  7.794e-05  53.311  < 2e-16 ***
## campaign           -9.218e-02  1.210e-02  -7.616 2.63e-14 ***
## poutcomeother       2.548e-01  1.051e-01   2.424 0.015362 *  
## poutcomesuccess     2.297e+00  9.504e-02  24.167  < 2e-16 ***
## poutcomeunknown    -1.413e-01  6.776e-02  -2.085 0.037110 *  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 22845  on 31648  degrees of freedom
## Residual deviance: 15125  on 31610  degrees of freedom
## AIC: 15203
## 
## Number of Fisher Scoring iterations: 6

vif(adj.model1)

##               GVIF Df GVIF^(1/(2*Df))
## job       2.949218 11        1.050389
## marital   1.187971  2        1.044002
## education 2.180470  3        1.138741
## balance   1.034920  1        1.017310
## housing   1.408584  1        1.186838
## loan      1.061581  1        1.030330
## contact   1.838442  2        1.164428
## day       1.344080  1        1.159345
## month     3.594816 11        1.059883
## duration  1.123002  1        1.059718
## campaign  1.104763  1        1.051077
## poutcome  1.226809  3        1.034656

fitControl = trainControl(method = "repeatedcv", number = 5, repeats = 1, classProbs = TRUE, summaryFunction = mnLogLoss)
fit.adj = train(factor(y) ~ ., data = training1, method = "glm", trControl = fitControl, metric = "logLoss")
summary(fit.adj$finalModel)

## 
## Call:
## NULL
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -5.7147  -0.3782  -0.2526  -0.1488   3.4359  
## 
## Coefficients:
##                      Estimate Std. Error z value Pr(>|z|)    
## (Intercept)        -2.384e+00  1.548e-01 -15.402  < 2e-16 ***
## `jobblue-collar`   -3.350e-01  8.784e-02  -3.814 0.000137 ***
## jobentrepreneur    -3.277e-01  1.504e-01  -2.178 0.029376 *  
## jobhousemaid       -5.708e-01  1.668e-01  -3.422 0.000622 ***
## jobmanagement      -1.302e-01  8.777e-02  -1.483 0.138063    
## jobretired          3.013e-01  1.037e-01   2.905 0.003669 ** 
## `jobself-employed` -1.618e-01  1.308e-01  -1.237 0.216015    
## jobservices        -2.574e-01  1.023e-01  -2.517 0.011850 *  
## jobstudent          4.419e-01  1.255e-01   3.521 0.000430 ***
## jobtechnician      -1.328e-01  8.255e-02  -1.609 0.107566    
## jobunemployed      -1.413e-01  1.322e-01  -1.069 0.284997    
## jobunknown         -1.151e-01  2.637e-01  -0.437 0.662452    
## maritalmarried     -1.780e-01  7.008e-02  -2.539 0.011103 *  
## maritalsingle       1.076e-01  7.548e-02   1.426 0.153845    
## educationsecondary  1.838e-01  7.758e-02   2.369 0.017832 *  
## educationtertiary   3.639e-01  8.938e-02   4.072 4.66e-05 ***
## educationunknown    1.135e-01  1.259e-01   0.902 0.367199    
## balance             1.606e-05  5.818e-06   2.761 0.005768 ** 
## housingyes         -7.357e-01  5.213e-02 -14.113  < 2e-16 ***
## loanyes            -4.653e-01  7.243e-02  -6.425 1.32e-10 ***
## contacttelephone   -1.132e-01  8.784e-02  -1.289 0.197366    
## contactunknown     -1.565e+00  8.770e-02 -17.847  < 2e-16 ***
## day                 7.628e-03  2.982e-03   2.558 0.010528 *  
## monthaug           -7.601e-01  9.374e-02  -8.109 5.12e-16 ***
## monthdec            6.124e-01  2.067e-01   2.962 0.003052 ** 
## monthfeb           -2.227e-01  1.063e-01  -2.096 0.036071 *  
## monthjan           -1.218e+00  1.421e-01  -8.571  < 2e-16 ***
## monthjul           -8.645e-01  9.265e-02  -9.332  < 2e-16 ***
## monthjun            3.015e-01  1.126e-01   2.677 0.007423 ** 
## monthmar            1.436e+00  1.466e-01   9.795  < 2e-16 ***
## monthmay           -4.449e-01  8.621e-02  -5.160 2.47e-07 ***
## monthnov           -8.733e-01  9.966e-02  -8.763  < 2e-16 ***
## monthoct            8.083e-01  1.284e-01   6.296 3.06e-10 ***
## monthsep            6.984e-01  1.418e-01   4.924 8.49e-07 ***
## duration            4.155e-03  7.794e-05  53.311  < 2e-16 ***
## campaign           -9.218e-02  1.210e-02  -7.616 2.63e-14 ***
## poutcomeother       2.548e-01  1.051e-01   2.424 0.015362 *  
## poutcomesuccess     2.297e+00  9.504e-02  24.167  < 2e-16 ***
## poutcomeunknown    -1.413e-01  6.776e-02  -2.085 0.037110 *  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 22845  on 31648  degrees of freedom
## Residual deviance: 15125  on 31610  degrees of freedom
## AIC: 15203
## 
## Number of Fisher Scoring iterations: 6

length(predictors(fit.adj))

## [1] 38

# Generate an ROC curve to help us determine the best threshold
fit.adj_predictions = predict(fit.adj, test, type = "prob")$yes
fit.adj_roc = roc(response = test$y, predictor = fit.adj_predictions, levels = c("no", "yes"))

## Setting direction: controls < cases

auc(fit.adj_roc)

## Area under the curve: 0.9087

plot(fit.adj_roc, print.thres = "best", col = "black")

# Predict
threshold = 0.093
fit.adj_predictions_categorical = factor(ifelse(predict(fit.adj, test, type = "prob")$yes >= threshold, "yes", "no"), levels = c("no", "yes"))
confusionMatrix(data = fit.adj_predictions_categorical, reference = factor(test$y, levels = c("no", "yes")))

## Confusion Matrix and Statistics
## 
##           Reference
## Prediction   no  yes
##        no  9565  190
##        yes 2411 1396
##                                           
##                Accuracy : 0.8082          
##                  95% CI : (0.8015, 0.8148)
##     No Information Rate : 0.8831          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0.4223          
##                                           
##  Mcnemar's Test P-Value : <2e-16          
##                                           
##             Sensitivity : 0.7987          
##             Specificity : 0.8802          
##          Pos Pred Value : 0.9805          
##          Neg Pred Value : 0.3667          
##              Prevalence : 0.8831          
##          Detection Rate : 0.7053          
##    Detection Prevalence : 0.7193          
##       Balanced Accuracy : 0.8394          
##                                           
##        'Positive' Class : no              
##

Penalized Regression/LASSO

Used in objective 2.

AUROC: 0.9102 AIC: 15201.86 Accuracy: 0.8289 Sensitivity: 0.8258 Specificity: 0.8525

However, once we try and increase the specificity, we realize that the ideal threshold for this base model is 0.107, which our accuracy is 0.8289.

fitControl=trainControl(method="repeatedcv",number=5,repeats=1) 

set.seed(124)

lambda=seq(0,.05,.001)
fit.glmnet = train(factor(y)~.,
                 data = training,
                 method = "glmnet",
                 trControl = fitControl,
                 tuneGrid = expand.grid(data.frame(alpha = 1, lambda = lambda)))
fit.glmnet

## glmnet 
## 
## 31649 samples
##    16 predictor
##     2 classes: 'no', 'yes' 
## 
## No pre-processing
## Resampling: Cross-Validated (5 fold, repeated 1 times) 
## Summary of sample sizes: 25320, 25318, 25319, 25319, 25320 
## Resampling results across tuning parameters:
## 
##   lambda  Accuracy   Kappa    
##   0.000   0.9019875  0.4022474
##   0.001   0.9022086  0.3992743
##   0.002   0.9023034  0.3957782
##   0.003   0.9028406  0.3946518
##   0.004   0.9028406  0.3889748
##   0.005   0.9020822  0.3792938
##   0.006   0.9013871  0.3702335
##   0.007   0.9012291  0.3648284
##   0.008   0.9011028  0.3600185
##   0.009   0.9009448  0.3560971
##   0.010   0.9009132  0.3525515
##   0.011   0.9005972  0.3467591
##   0.012   0.9001548  0.3402719
##   0.013   0.9000916  0.3364585
##   0.014   0.8999652  0.3328241
##   0.015   0.8997756  0.3274568
##   0.016   0.8996176  0.3229941
##   0.017   0.8992385  0.3177516
##   0.018   0.8987013  0.3099459
##   0.019   0.8983538  0.3044727
##   0.020   0.8979115  0.2970359
##   0.021   0.8975007  0.2902943
##   0.022   0.8969951  0.2831910
##   0.023   0.8964896  0.2760122
##   0.024   0.8963000  0.2712149
##   0.025   0.8962052  0.2674412
##   0.026   0.8955101  0.2579662
##   0.027   0.8953521  0.2528487
##   0.028   0.8946886  0.2426713
##   0.029   0.8942146  0.2345472
##   0.030   0.8938670  0.2272993
##   0.031   0.8934246  0.2185806
##   0.032   0.8929823  0.2117265
##   0.033   0.8923188  0.2013404
##   0.034   0.8917817  0.1928894
##   0.035   0.8913709  0.1855636
##   0.036   0.8909286  0.1783645
##   0.037   0.8905178  0.1715052
##   0.038   0.8903914  0.1662652
##   0.039   0.8899175  0.1588041
##   0.040   0.8899175  0.1547399
##   0.041   0.8895699  0.1487580
##   0.042   0.8893488  0.1443191
##   0.043   0.8890644  0.1387166
##   0.044   0.8887168  0.1332446
##   0.045   0.8885273  0.1298093
##   0.046   0.8882429  0.1247858
##   0.047   0.8883061  0.1226102
##   0.048   0.8879585  0.1163336
##   0.049   0.8875162  0.1094384
##   0.050   0.8874214  0.1074797
## 
## Tuning parameter 'alpha' was held constant at a value of 1
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were alpha = 1 and lambda = 0.003.

plot(fit.glmnet)

opt.pen= fit.glmnet$finalModel$lambdaOpt
coef(fit.glmnet$finalModel, opt.pen)

## 43 x 1 sparse Matrix of class "dgCMatrix"
##                               s1
## (Intercept)        -2.552925e+00
## age                 .           
## jobblue-collar     -1.579392e-01
## jobentrepreneur     .           
## jobhousemaid       -1.951650e-01
## jobmanagement       .           
## jobretired          2.994994e-01
## jobself-employed    .           
## jobservices         .           
## jobstudent          5.222552e-01
## jobtechnician       .           
## jobunemployed       .           
## jobunknown          .           
## maritalmarried     -1.364113e-01
## maritalsingle       1.009321e-01
## educationsecondary  .           
## educationtertiary   1.481663e-01
## educationunknown    .           
## defaultyes          .           
## balance             1.019485e-05
## housingyes         -6.457751e-01
## loanyes            -3.557147e-01
## contacttelephone    .           
## contactunknown     -1.226280e+00
## day                 .           
## monthaug           -3.186051e-01
## monthdec            7.343465e-01
## monthfeb            .           
## monthjan           -5.355335e-01
## monthjul           -4.300466e-01
## monthjun            2.636016e-01
## monthmar            1.611861e+00
## monthmay           -1.457283e-01
## monthnov           -3.724500e-01
## monthoct            1.050138e+00
## monthsep            8.888944e-01
## duration            3.865347e-03
## campaign           -6.197404e-02
## pdays               .           
## previous            5.945810e-03
## poutcomeother       1.170247e-01
## poutcomesuccess     2.196243e+00
## poutcomeunknown    -1.720155e-01

training2 = training[,-c(1,5,14)]
pen.model1 = glm(factor(y)~., data = training2, family = "binomial")
summary(pen.model1)

## 
## Call:
## glm(formula = factor(y) ~ ., family = "binomial", data = training2)
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -5.7145  -0.3781  -0.2524  -0.1488   3.4404  
## 
## Coefficients:
##                      Estimate Std. Error z value Pr(>|z|)    
## (Intercept)        -2.417e+00  1.563e-01 -15.466  < 2e-16 ***
## jobblue-collar     -3.333e-01  8.785e-02  -3.794 0.000148 ***
## jobentrepreneur    -3.258e-01  1.504e-01  -2.166 0.030321 *  
## jobhousemaid       -5.688e-01  1.668e-01  -3.411 0.000648 ***
## jobmanagement      -1.316e-01  8.779e-02  -1.499 0.133920    
## jobretired          3.032e-01  1.037e-01   2.923 0.003462 ** 
## jobself-employed   -1.617e-01  1.308e-01  -1.237 0.216202    
## jobservices        -2.555e-01  1.023e-01  -2.497 0.012513 *  
## jobstudent          4.415e-01  1.255e-01   3.517 0.000437 ***
## jobtechnician      -1.328e-01  8.256e-02  -1.609 0.107719    
## jobunemployed      -1.406e-01  1.322e-01  -1.063 0.287570    
## jobunknown         -1.155e-01  2.638e-01  -0.438 0.661557    
## maritalmarried     -1.790e-01  7.008e-02  -2.554 0.010644 *  
## maritalsingle       1.074e-01  7.547e-02   1.423 0.154672    
## educationsecondary  1.850e-01  7.758e-02   2.385 0.017099 *  
## educationtertiary   3.649e-01  8.938e-02   4.083 4.45e-05 ***
## educationunknown    1.153e-01  1.259e-01   0.916 0.359616    
## balance             1.606e-05  5.817e-06   2.761 0.005757 ** 
## housingyes         -7.372e-01  5.215e-02 -14.136  < 2e-16 ***
## loanyes            -4.654e-01  7.242e-02  -6.427 1.30e-10 ***
## contacttelephone   -1.148e-01  8.786e-02  -1.306 0.191489    
## contactunknown     -1.566e+00  8.769e-02 -17.855  < 2e-16 ***
## day                 7.733e-03  2.982e-03   2.593 0.009517 ** 
## monthaug           -7.619e-01  9.374e-02  -8.128 4.38e-16 ***
## monthdec            6.101e-01  2.068e-01   2.950 0.003180 ** 
## monthfeb           -2.265e-01  1.063e-01  -2.130 0.033151 *  
## monthjan           -1.223e+00  1.421e-01  -8.602  < 2e-16 ***
## monthjul           -8.670e-01  9.265e-02  -9.357  < 2e-16 ***
## monthjun            2.997e-01  1.126e-01   2.661 0.007787 ** 
## monthmar            1.434e+00  1.466e-01   9.779  < 2e-16 ***
## monthmay           -4.458e-01  8.621e-02  -5.171 2.33e-07 ***
## monthnov           -8.755e-01  9.967e-02  -8.784  < 2e-16 ***
## monthoct            8.048e-01  1.284e-01   6.268 3.65e-10 ***
## monthsep            6.960e-01  1.419e-01   4.906 9.29e-07 ***
## duration            4.155e-03  7.794e-05  53.315  < 2e-16 ***
## campaign           -9.270e-02  1.212e-02  -7.649 2.03e-14 ***
## previous            1.170e-02  7.429e-03   1.575 0.115164    
## poutcomeother       2.402e-01  1.056e-01   2.275 0.022891 *  
## poutcomesuccess     2.295e+00  9.504e-02  24.147  < 2e-16 ***
## poutcomeunknown    -1.072e-01  7.121e-02  -1.505 0.132346    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 22845  on 31648  degrees of freedom
## Residual deviance: 15122  on 31609  degrees of freedom
## AIC: 15202
## 
## Number of Fisher Scoring iterations: 6

vif(pen.model1)

##               GVIF Df GVIF^(1/(2*Df))
## job       2.951293 11        1.050423
## marital   1.188100  2        1.044031
## education 2.180369  3        1.138732
## balance   1.034862  1        1.017282
## housing   1.409205  1        1.187100
## loan      1.061566  1        1.030323
## contact   1.838422  2        1.164424
## day       1.344355  1        1.159463
## month     3.595446 11        1.059892
## duration  1.123020  1        1.059727
## campaign  1.105342  1        1.051352
## previous  1.264204  1        1.124368
## poutcome  1.497659  3        1.069635

# Generate an ROC curve to help us determine the best threshold
fit.glmnet_predictions = predict(fit.glmnet, test, type = "prob")$yes
fit.glmnet_roc = roc(response = test$y, predictor = fit.glmnet_predictions, levels = c("no", "yes"))

## Setting direction: controls < cases

auc(fit.glmnet_roc)

## Area under the curve: 0.9102

plot(fit.glmnet_roc, print.thres = "best", col = "black")

# Predict
threshold = 0.107
fit.glmnet_predictions_categorical = factor(ifelse(predict(fit.glmnet, test, type = "prob")$yes >= threshold, "yes", "no"), levels = c("no", "yes"))
confusionMatrix(data = fit.glmnet_predictions_categorical, reference = factor(test$y, levels = c("no", "yes")))

## Confusion Matrix and Statistics
## 
##           Reference
## Prediction   no  yes
##        no  9890  234
##        yes 2086 1352
##                                           
##                Accuracy : 0.8289          
##                  95% CI : (0.8225, 0.8352)
##     No Information Rate : 0.8831          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0.4502          
##                                           
##  Mcnemar's Test P-Value : <2e-16          
##                                           
##             Sensitivity : 0.8258          
##             Specificity : 0.8525          
##          Pos Pred Value : 0.9769          
##          Neg Pred Value : 0.3933          
##              Prevalence : 0.8831          
##          Detection Rate : 0.7292          
##    Detection Prevalence : 0.7465          
##       Balanced Accuracy : 0.8391          
##                                           
##        'Positive' Class : no              
##

Stepwise Feature Selection

AUROC: 0.9087 AIC: 15201.86 Accuracy: 0.8087 Sensitivity: 0.7991 Specificity: 0.8808

Note that because we have such unbalanced data, our accuracy is always expected to be high. Only 10% of the results were “yes”.

However, once we try and increase the specificity, we realize that the ideal threshold for this base model is 0.093, which our accuracy is 0.8087.

set.seed(124)
fit.step  = glm(factor(y) ~., data = training, family = binomial) %>%
  stepAIC(trace = FALSE)
summary(fit.step)

## 
## Call:
## glm(formula = factor(y) ~ job + marital + education + balance + 
##     housing + loan + contact + day + month + duration + campaign + 
##     previous + poutcome, family = binomial, data = training)
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -5.7145  -0.3781  -0.2524  -0.1488   3.4404  
## 
## Coefficients:
##                      Estimate Std. Error z value Pr(>|z|)    
## (Intercept)        -2.417e+00  1.563e-01 -15.466  < 2e-16 ***
## jobblue-collar     -3.333e-01  8.785e-02  -3.794 0.000148 ***
## jobentrepreneur    -3.258e-01  1.504e-01  -2.166 0.030321 *  
## jobhousemaid       -5.688e-01  1.668e-01  -3.411 0.000648 ***
## jobmanagement      -1.316e-01  8.779e-02  -1.499 0.133920    
## jobretired          3.032e-01  1.037e-01   2.923 0.003462 ** 
## jobself-employed   -1.617e-01  1.308e-01  -1.237 0.216202    
## jobservices        -2.555e-01  1.023e-01  -2.497 0.012513 *  
## jobstudent          4.415e-01  1.255e-01   3.517 0.000437 ***
## jobtechnician      -1.328e-01  8.256e-02  -1.609 0.107719    
## jobunemployed      -1.406e-01  1.322e-01  -1.063 0.287570    
## jobunknown         -1.155e-01  2.638e-01  -0.438 0.661557    
## maritalmarried     -1.790e-01  7.008e-02  -2.554 0.010644 *  
## maritalsingle       1.074e-01  7.547e-02   1.423 0.154672    
## educationsecondary  1.850e-01  7.758e-02   2.385 0.017099 *  
## educationtertiary   3.649e-01  8.938e-02   4.083 4.45e-05 ***
## educationunknown    1.153e-01  1.259e-01   0.916 0.359616    
## balance             1.606e-05  5.817e-06   2.761 0.005757 ** 
## housingyes         -7.372e-01  5.215e-02 -14.136  < 2e-16 ***
## loanyes            -4.654e-01  7.242e-02  -6.427 1.30e-10 ***
## contacttelephone   -1.148e-01  8.786e-02  -1.306 0.191489    
## contactunknown     -1.566e+00  8.769e-02 -17.855  < 2e-16 ***
## day                 7.733e-03  2.982e-03   2.593 0.009517 ** 
## monthaug           -7.619e-01  9.374e-02  -8.128 4.38e-16 ***
## monthdec            6.101e-01  2.068e-01   2.950 0.003180 ** 
## monthfeb           -2.265e-01  1.063e-01  -2.130 0.033151 *  
## monthjan           -1.223e+00  1.421e-01  -8.602  < 2e-16 ***
## monthjul           -8.670e-01  9.265e-02  -9.357  < 2e-16 ***
## monthjun            2.997e-01  1.126e-01   2.661 0.007787 ** 
## monthmar            1.434e+00  1.466e-01   9.779  < 2e-16 ***
## monthmay           -4.458e-01  8.621e-02  -5.171 2.33e-07 ***
## monthnov           -8.755e-01  9.967e-02  -8.784  < 2e-16 ***
## monthoct            8.048e-01  1.284e-01   6.268 3.65e-10 ***
## monthsep            6.960e-01  1.419e-01   4.906 9.29e-07 ***
## duration            4.155e-03  7.794e-05  53.315  < 2e-16 ***
## campaign           -9.270e-02  1.212e-02  -7.649 2.03e-14 ***
## previous            1.170e-02  7.429e-03   1.575 0.115164    
## poutcomeother       2.402e-01  1.056e-01   2.275 0.022891 *  
## poutcomesuccess     2.295e+00  9.504e-02  24.147  < 2e-16 ***
## poutcomeunknown    -1.072e-01  7.121e-02  -1.505 0.132346    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 22845  on 31648  degrees of freedom
## Residual deviance: 15122  on 31609  degrees of freedom
## AIC: 15202
## 
## Number of Fisher Scoring iterations: 6

coef(fit.step)

##        (Intercept)     jobblue-collar    jobentrepreneur       jobhousemaid 
##      -2.416939e+00      -3.333024e-01      -3.257645e-01      -5.687815e-01 
##      jobmanagement         jobretired   jobself-employed        jobservices 
##      -1.315824e-01       3.032136e-01      -1.617472e-01      -2.554693e-01 
##         jobstudent      jobtechnician      jobunemployed         jobunknown 
##       4.414831e-01      -1.327954e-01      -1.405869e-01      -1.154612e-01 
##     maritalmarried      maritalsingle educationsecondary  educationtertiary 
##      -1.789881e-01       1.074058e-01       1.849937e-01       3.649409e-01 
##   educationunknown            balance         housingyes            loanyes 
##       1.153179e-01       1.606222e-05      -7.371729e-01      -4.654309e-01 
##   contacttelephone     contactunknown                day           monthaug 
##      -1.147624e-01      -1.565769e+00       7.733248e-03      -7.618837e-01 
##           monthdec           monthfeb           monthjan           monthjul 
##       6.100588e-01      -2.264698e-01      -1.222727e+00      -8.669716e-01 
##           monthjun           monthmar           monthmay           monthnov 
##       2.997300e-01       1.434038e+00      -4.457667e-01      -8.755030e-01 
##           monthoct           monthsep           duration           campaign 
##       8.048252e-01       6.960106e-01       4.155302e-03      -9.270440e-02 
##           previous      poutcomeother    poutcomesuccess    poutcomeunknown 
##       1.170318e-02       2.401987e-01       2.295025e+00      -1.071710e-01

vif(fit.step)

##               GVIF Df GVIF^(1/(2*Df))
## job       2.951293 11        1.050423
## marital   1.188100  2        1.044031
## education 2.180369  3        1.138732
## balance   1.034862  1        1.017282
## housing   1.409205  1        1.187100
## loan      1.061566  1        1.030323
## contact   1.838422  2        1.164424
## day       1.344355  1        1.159463
## month     3.595446 11        1.059892
## duration  1.123020  1        1.059727
## campaign  1.105342  1        1.051352
## previous  1.264204  1        1.124368
## poutcome  1.497659  3        1.069635

# Generate an ROC curve to help us determine the best threshold
fit.step_predictions = predict(fit.step, test, type = "response")
fit.step_roc = roc(response = test$y, predictor = fit.step_predictions, levels = c("no", "yes"))

## Setting direction: controls < cases

auc(fit.step_roc)

## Area under the curve: 0.9087

plot(fit.step_roc, print.thres = "best", col = "black")

# Predict
threshold = 0.093
fit.step_predictions_categorical = factor(ifelse(predict(fit.step, test, "response") >= threshold, "yes", "no"), levels = c("no", "yes"))
confusionMatrix(data = fit.step_predictions_categorical, reference = factor(test$y, levels = c("no", "yes")))

## Confusion Matrix and Statistics
## 
##           Reference
## Prediction   no  yes
##        no  9570  189
##        yes 2406 1397
##                                           
##                Accuracy : 0.8087          
##                  95% CI : (0.8019, 0.8152)
##     No Information Rate : 0.8831          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0.4233          
##                                           
##  Mcnemar's Test P-Value : <2e-16          
##                                           
##             Sensitivity : 0.7991          
##             Specificity : 0.8808          
##          Pos Pred Value : 0.9806          
##          Neg Pred Value : 0.3673          
##              Prevalence : 0.8831          
##          Detection Rate : 0.7056          
##    Detection Prevalence : 0.7196          
##       Balanced Accuracy : 0.8400          
##                                           
##        'Positive' Class : no              
##

Manual Stepwise

NOTE: The final model was produced elsewhere through a manual stepping process as some of the automated routines failed

AUROC: 0.9045 AIC: 15484.51 Accuracy: 0.8188 Sensitivity: 0.8137 Specificity: 0.8569 Prevalence: 0.8831 PPV: 0.9806 NPV: 0.3673

Note that because we have such unbalanced data, our accuracy is always expected to be high. Only 10% of the results were “yes”.

However, once we try and increase the specificity, we realize that the ideal threshold for this base model is 0.093, which our accuracy is 0.8087.

set.seed(124)
stepwiseResults = read.csv("Stepwise Regression Results.csv")

ggplot(stepwiseResults, aes(x = reorder(Variable.Added, AIC, decreasing = T), y = AIC)) + 
  geom_point(size = 3) +
  labs(title = "AIC Metric with Additional Variables being Added", x= "Variable Added", y = "AIC") +
  theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1))

logit_step = glm(factor(y) ~ housing + contact + month + duration + poutcome , data = training, family = "binomial")
summary(logit_step)

## 
## Call:
## glm(formula = factor(y) ~ housing + contact + month + duration + 
##     poutcome, family = "binomial", data = training)
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -5.6143  -0.3832  -0.2706  -0.1604   3.3708  
## 
## Coefficients:
##                    Estimate Std. Error z value Pr(>|z|)    
## (Intercept)      -2.304e+00  9.292e-02 -24.792  < 2e-16 ***
## housingyes       -8.748e-01  5.049e-02 -17.327  < 2e-16 ***
## contacttelephone -1.907e-01  8.488e-02  -2.247 0.024633 *  
## contactunknown   -1.672e+00  8.716e-02 -19.185  < 2e-16 ***
## monthaug         -9.394e-01  9.023e-02 -10.411  < 2e-16 ***
## monthdec          6.813e-01  2.058e-01   3.310 0.000933 ***
## monthfeb         -3.426e-01  9.894e-02  -3.462 0.000536 ***
## monthjan         -1.158e+00  1.387e-01  -8.353  < 2e-16 ***
## monthjul         -1.107e+00  9.048e-02 -12.230  < 2e-16 ***
## monthjun          2.159e-01  1.091e-01   1.979 0.047871 *  
## monthmar          1.501e+00  1.435e-01  10.460  < 2e-16 ***
## monthmay         -4.852e-01  8.407e-02  -5.771 7.90e-09 ***
## monthnov         -9.130e-01  9.853e-02  -9.266  < 2e-16 ***
## monthoct          9.236e-01  1.279e-01   7.219 5.23e-13 ***
## monthsep          7.666e-01  1.397e-01   5.487 4.08e-08 ***
## duration          4.106e-03  7.647e-05  53.697  < 2e-16 ***
## poutcomeother     2.501e-01  1.038e-01   2.410 0.015948 *  
## poutcomesuccess   2.385e+00  9.430e-02  25.294  < 2e-16 ***
## poutcomeunknown  -1.530e-01  6.737e-02  -2.272 0.023104 *  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 22845  on 31648  degrees of freedom
## Residual deviance: 15447  on 31630  degrees of freedom
## AIC: 15485
## 
## Number of Fisher Scoring iterations: 6

vif(logit_step)

##              GVIF Df GVIF^(1/(2*Df))
## housing  1.340648  1        1.157864
## contact  1.772873  2        1.153903
## month    2.345615 11        1.039513
## duration 1.105390  1        1.051376
## poutcome 1.222927  3        1.034110

# Generate an ROC curve to help us determine the best threshold
logit_step_predictions = predict(logit_step, test, type = "response")
logit_step_roc = roc(response = test$y, predictor = logit_step_predictions, levels = c("no", "yes"))

## Setting direction: controls < cases

auc(logit_step_roc)

## Area under the curve: 0.9045

plot(logit_step_roc, print.thres = "best", col = "black")

# Predict
threshold = 0.101
logit_step_predictions_categorical = factor(ifelse(predict(logit_step, test, "response") >= threshold, "yes", "no"), levels = c("no", "yes"))
confusionMatrix(data = logit_step_predictions_categorical, reference = factor(test$y, levels = c("no", "yes")))

## Confusion Matrix and Statistics
## 
##           Reference
## Prediction   no  yes
##        no  9745  227
##        yes 2231 1359
##                                           
##                Accuracy : 0.8188          
##                  95% CI : (0.8122, 0.8252)
##     No Information Rate : 0.8831          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0.4332          
##                                           
##  Mcnemar's Test P-Value : <2e-16          
##                                           
##             Sensitivity : 0.8137          
##             Specificity : 0.8569          
##          Pos Pred Value : 0.9772          
##          Neg Pred Value : 0.3786          
##              Prevalence : 0.8831          
##          Detection Rate : 0.7186          
##    Detection Prevalence : 0.7353          
##       Balanced Accuracy : 0.8353          
##                                           
##        'Positive' Class : no              
##

AIC Comparison

AIC(model1)

## [1] 15206.49

AIC(adj.model1)

## [1] 15202.89

AIC(pen.model1)

## [1] 15201.86

AIC(fit.step)

## [1] 15201.86

AIC(logit_step)

## [1] 15484.51

library(gtsummary)

## 
## Attaching package: 'gtsummary'

## The following object is masked from 'package:MASS':
## 
##     select

# Create a nice looking table
tbl_regression(logit_step, exponentiate = TRUE) %>% 
  as_gt() %>% 
  gt::tab_options(table.font.size = 10)

Characteristic	OR¹	95% CI¹	p-value
housing
no	—	—
yes	0.42	0.38, 0.46	<0.001
contact
cellular	—	—
telephone	0.83	0.70, 0.97	0.025
unknown	0.19	0.16, 0.22	<0.001
month
apr	—	—
aug	0.39	0.33, 0.47	<0.001
dec	1.98	1.32, 2.95	<0.001
feb	0.71	0.58, 0.86	<0.001
jan	0.31	0.24, 0.41	<0.001
jul	0.33	0.28, 0.39	<0.001
jun	1.24	1.00, 1.54	0.048
mar	4.49	3.38, 5.94	<0.001
may	0.62	0.52, 0.73	<0.001
nov	0.40	0.33, 0.49	<0.001
oct	2.52	1.96, 3.23	<0.001
sep	2.15	1.64, 2.83	<0.001
duration	1.00	1.00, 1.00	<0.001
poutcome
failure	—	—
other	1.28	1.05, 1.57	0.016
success	10.9	9.04, 13.1	<0.001
unknown	0.86	0.75, 0.98	0.023
¹ OR = Odds Ratio, CI = Confidence Interval

Objective 2

EDA

# make new numeric column for "y" column
bank$y_num = num(ifelse(bank$y == "yes",1,0))

## Age
ggplot(bank,aes(x=age,y=y_num, color = marital))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE) + ggtitle("Loess Plot for Age vs. Subscriptions vs. Marital Status")

## `geom_smooth()` using formula 'y ~ x'

ggplot(bank,aes(x=age,y=y_num, color = education))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE) + ggtitle("Loess Plot for Age vs. Subscriptions vs. Education") # no difference

## `geom_smooth()` using formula 'y ~ x'

ggplot(bank,aes(x=age,y=y_num, color = housing))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE) + ggtitle("Loess Plot for Age vs. Subscriptions vs. Housing")

## `geom_smooth()` using formula 'y ~ x'

ggplot(bank,aes(x=age,y=y_num, color = loan))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE)+ ggtitle("Loess Plot for Age vs. Subscriptions vs. Loan")

## `geom_smooth()` using formula 'y ~ x'

## Balance
ggplot(bank,aes(x=balance,y=y_num, color = marital))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE) + ggtitle("Loess Plot for Balance vs. Subscriptions vs. Marital")

## `geom_smooth()` using formula 'y ~ x'

ggplot(bank,aes(x=balance,y=y_num, color = housing))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE) + ggtitle("Loess Plot for Balance vs. Subscriptions vs. Housing")# no difference

## `geom_smooth()` using formula 'y ~ x'

ggplot(bank,aes(x=balance,y=y_num, color = education))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE) + ggtitle("Loess Plot for Balance vs. Subscriptions vs. Education")

## `geom_smooth()` using formula 'y ~ x'

ggplot(bank,aes(x=balance,y=y_num, color = month))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE) + ggtitle("Loess Plot for Balance vs. Subscriptions vs. Loan")

## `geom_smooth()` using formula 'y ~ x'

## Duration
bank2 = bank 
bank2$dur.poly <- poly(bank2$duration^2)

ggplot(bank, aes(x = duration, y = y_num)) +
  geom_point() +
  geom_smooth(method = "loess", size = 1, span = 3, se = FALSE) +
  facet_wrap(~marital)+
  ggtitle("Loess Plot for Duration vs. Subscriptions vs. Marital")

## `geom_smooth()` using formula 'y ~ x'

ggplot(bank,aes(x=duration,y=y_num))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE)+ 
  facet_wrap(~housing) +
  ggtitle("Loess Plot for Duration vs. Subscriptions vs. Housing")

## `geom_smooth()` using formula 'y ~ x'

ggplot(bank,aes(x=duration,y=y_num, color = education))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE)+ ggtitle("Loess Plot for Duration vs. Subscriptions vs. Education")

## `geom_smooth()` using formula 'y ~ x'

ggplot(bank,aes(x=duration,y=y_num, color = job))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE) + ggtitle("Loess Plot for Duration vs. Subscriptions vs. Job")

## `geom_smooth()` using formula 'y ~ x'

ggplot(bank,aes(x=duration,y=y_num, color = poutcome))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE) + ggtitle("Loess Plot for Duration vs. Subscriptions vs. Poutcome")

## `geom_smooth()` using formula 'y ~ x'

ggplot(bank,aes(x=duration,y=y_num, color = month))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE) + ggtitle("Loess Plot for Duration vs. Subscriptions vs. Month")

## `geom_smooth()` using formula 'y ~ x'

ggplot(bank,aes(x=duration,y=y_num))+geom_point()+
  geom_smooth(method="loess",size=1,span=.75, se = FALSE) + ggtitle("Loess Plot for Duration vs. Subscriptions")

## `geom_smooth()` using formula 'y ~ x'

## Pdays
ggplot(bank,aes(x=pdays,y=y_num, color = marital))+geom_point()+
  geom_smooth(method="loess",size=1,span=1, se = FALSE) + ggtitle("Loess Plot for Previous Days vs. Subscriptions vs. Marital")

## `geom_smooth()` using formula 'y ~ x'

ggplot(bank,aes(x=pdays,y=y_num, color = housing))+geom_point()+
  geom_smooth(method="loess",size=1,span=1, se = FALSE) + ggtitle("Loess Plot for Previous Days vs. Subscriptions vs. Housing")

## `geom_smooth()` using formula 'y ~ x'

ggplot(bank,aes(x=pdays,y=y_num, color = education))+geom_point()+
  geom_smooth(method="loess",size=1,span=1, se = FALSE) + ggtitle("Loess Plot for Previous Days vs. Subscriptions vs. Education")

## `geom_smooth()` using formula 'y ~ x'

ggplot(bank,aes(x=pdays,y=y_num, color = loan))+geom_point()+
  geom_smooth(method="loess",size=1,span=1, se = FALSE) + ggtitle("Loess Plot for Previous Days vs. Subscriptions vs. Loan")

## `geom_smooth()` using formula 'y ~ x'

bank = bank[,-c(18)] # remove y_num from dataset

ggplot(bank, aes(x = poutcome, y = duration, fill = y)) +
  geom_boxplot() + ggtitle("Boxplot for Previous Outcome vs. Duration by Subscription Outcome")

ggplot(bank, aes(x =month, y = duration, fill = y)) +
  geom_boxplot() + ggtitle("Boxplot for Month vs. Duration by Subscription Outcome")

ggplot(bank, aes(x =marital, y = duration, fill = y)) +
  geom_boxplot() + ggtitle("Boxplot for Marital Status vs. Duration by Subscription Outcome")

ggplot(bank, aes(x = housing, y = duration, fill = y)) +
  geom_boxplot() + ggtitle("Boxplot for Housing vs. Duration by Subscription Outcome")

ggplot(bank, aes(x = loan, y = duration, fill = y)) +
  geom_boxplot() + ggtitle("Boxplot for Housing vs. Duration by Subscription Outcome")

ggplot(bank, aes(x =month, y = log(balance), fill = y)) +
  geom_boxplot() + ggtitle("Boxplot for Month vs. log(Balance) by Subscription Outcome")

## Warning in log(balance): NaNs produced

## Warning in log(balance): NaNs produced

## Warning: Removed 7280 rows containing non-finite values (stat_boxplot).

ggplot(bank, aes(x = housing, y = log(balance), fill = y)) +
  geom_boxplot() + ggtitle("Boxplot for Housing vs. log(Balance) by Subscription Outcome")

## Warning in log(balance): NaNs produced

## Warning in log(balance): NaNs produced

## Warning: Removed 7280 rows containing non-finite values (stat_boxplot).

ggplot(bank, aes(x = housing, y = log(campaign), fill = y)) +
  geom_boxplot() + ggtitle("Boxplot for Housing vs. log(Campaign) by Subscription Outcome")

ggplot(bank, aes(x = month, y = log(balance), fill = y)) +
  geom_boxplot() + ggtitle("Boxplot for Month vs. Balance by Subscription Outcome")

## Warning in log(balance): NaNs produced

## Warning in log(balance): NaNs produced

## Warning: Removed 7280 rows containing non-finite values (stat_boxplot).

ggplot(bank, aes(x = poutcome, y = log(previous), fill = y)) +
  geom_boxplot() + ggtitle("Boxplot for Previous Outcome vs. log(Previous) by Subscription Outcome")

## Warning: Removed 36954 rows containing non-finite values (stat_boxplot).

1. More complicated logistic regression model

Please refer to the penalized regression/LASSO to see how we determine these variables.

AUROC: 0.9147 AIC: 14222.29 Accuracy: 0.8389
Sensitivity: 0.8378
Specificity: 0.8474
Prevalence: 0.8831 PPV: 0.9764
NPV: 0.4089

logit_complex = glm(factor(y) ~ poutcome +  poly(duration,2) + poutcome:poly(duration,2) + month + month:poly(duration,2) + contact + housing + housing:poly(duration,2) + job + campaign + loan + job:poly(duration,2) + marital + education + day + balance + month:balance, 
                    data = training, family = "binomial")

summary(logit_complex)

## 
## Call:
## glm(formula = factor(y) ~ poutcome + poly(duration, 2) + poutcome:poly(duration, 
##     2) + month + month:poly(duration, 2) + contact + housing + 
##     housing:poly(duration, 2) + job + campaign + loan + job:poly(duration, 
##     2) + marital + education + day + balance + month:balance, 
##     family = "binomial", data = training)
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -2.9323  -0.3564  -0.1893  -0.0834   4.1629  
## 
## Coefficients:
##                                       Estimate Std. Error z value Pr(>|z|)    
## (Intercept)                         -1.188e+00  1.723e-01  -6.895 5.39e-12 ***
## poutcomeother                        3.028e-01  1.238e-01   2.446 0.014432 *  
## poutcomesuccess                      2.330e+00  1.089e-01  21.392  < 2e-16 ***
## poutcomeunknown                     -1.879e-01  8.031e-02  -2.339 0.019320 *  
## poly(duration, 2)1                   6.803e+01  1.793e+01   3.795 0.000148 ***
## poly(duration, 2)2                  -1.280e+02  2.807e+01  -4.559 5.13e-06 ***
## monthaug                            -1.169e+00  1.204e-01  -9.710  < 2e-16 ***
## monthdec                             3.475e-01  2.644e-01   1.314 0.188788    
## monthfeb                            -2.348e-01  1.247e-01  -1.883 0.059702 .  
## monthjan                            -1.524e+00  1.851e-01  -8.236  < 2e-16 ***
## monthjul                            -1.270e+00  1.282e-01  -9.902  < 2e-16 ***
## monthjun                             2.582e-01  1.366e-01   1.889 0.058861 .  
## monthmar                             1.516e+00  1.886e-01   8.037 9.20e-16 ***
## monthmay                            -1.153e+00  1.247e-01  -9.244  < 2e-16 ***
## monthnov                            -9.008e-01  1.278e-01  -7.051 1.77e-12 ***
## monthoct                             8.102e-01  1.501e-01   5.397 6.78e-08 ***
## monthsep                             6.723e-01  1.749e-01   3.844 0.000121 ***
## contacttelephone                    -5.148e-02  8.902e-02  -0.578 0.563077    
## contactunknown                      -1.820e+00  9.850e-02 -18.474  < 2e-16 ***
## housingyes                          -9.081e-01  6.601e-02 -13.756  < 2e-16 ***
## jobblue-collar                      -5.384e-01  1.173e-01  -4.589 4.45e-06 ***
## jobentrepreneur                     -5.291e-01  2.096e-01  -2.525 0.011578 *  
## jobhousemaid                        -4.387e-01  1.957e-01  -2.241 0.025029 *  
## jobmanagement                       -1.051e-01  9.995e-02  -1.051 0.293244    
## jobretired                           2.508e-01  1.201e-01   2.088 0.036820 *  
## jobself-employed                    -2.000e-01  1.612e-01  -1.240 0.214846    
## jobservices                         -2.515e-01  1.304e-01  -1.929 0.053762 .  
## jobstudent                           3.754e-01  1.440e-01   2.606 0.009154 ** 
## jobtechnician                       -1.343e-01  9.929e-02  -1.353 0.176092    
## jobunemployed                       -2.014e-01  1.652e-01  -1.219 0.222776    
## jobunknown                          -2.344e-01  3.833e-01  -0.612 0.540777    
## campaign                            -9.333e-02  1.253e-02  -7.451 9.27e-14 ***
## loanyes                             -4.799e-01  7.387e-02  -6.496 8.23e-11 ***
## maritalmarried                      -1.816e-01  7.184e-02  -2.527 0.011494 *  
## maritalsingle                        1.008e-01  7.763e-02   1.298 0.194225    
## educationsecondary                   1.618e-01  7.899e-02   2.049 0.040474 *  
## educationtertiary                    3.468e-01  9.098e-02   3.812 0.000138 ***
## educationunknown                     3.252e-02  1.296e-01   0.251 0.801893    
## day                                  8.481e-03  3.077e-03   2.756 0.005850 ** 
## balance                              2.416e-05  2.105e-05   1.148 0.250993    
## poutcomeother:poly(duration, 2)1    -7.120e+00  1.705e+01  -0.418 0.676300    
## poutcomesuccess:poly(duration, 2)1  -6.029e+00  2.261e+01  -0.267 0.789733    
## poutcomeunknown:poly(duration, 2)1   5.386e+01  1.126e+01   4.784 1.72e-06 ***
## poutcomeother:poly(duration, 2)2     2.105e+00  2.952e+01   0.071 0.943160    
## poutcomesuccess:poly(duration, 2)2   1.677e+01  3.438e+01   0.488 0.625645    
## poutcomeunknown:poly(duration, 2)2   6.646e+01  1.828e+01   3.636 0.000277 ***
## poly(duration, 2)1:monthaug          8.180e+01  1.431e+01   5.717 1.08e-08 ***
## poly(duration, 2)2:monthaug         -8.609e+01  2.301e+01  -3.741 0.000183 ***
## poly(duration, 2)1:monthdec          6.822e+01  4.052e+01   1.684 0.092236 .  
## poly(duration, 2)2:monthdec         -4.307e+01  5.465e+01  -0.788 0.430628    
## poly(duration, 2)1:monthfeb          1.641e+01  1.534e+01   1.070 0.284569    
## poly(duration, 2)2:monthfeb          4.569e+01  2.211e+01   2.066 0.038824 *  
## poly(duration, 2)1:monthjan          4.461e+01  2.052e+01   2.174 0.029672 *  
## poly(duration, 2)2:monthjan          1.126e+01  2.485e+01   0.453 0.650581    
## poly(duration, 2)1:monthjul          8.500e+01  1.386e+01   6.132 8.69e-10 ***
## poly(duration, 2)2:monthjul         -2.574e+01  2.123e+01  -1.212 0.225369    
## poly(duration, 2)1:monthjun          8.988e+01  1.479e+01   6.078 1.22e-09 ***
## poly(duration, 2)2:monthjun          3.734e+00  2.022e+01   0.185 0.853482    
## poly(duration, 2)1:monthmar          6.582e+01  4.238e+01   1.553 0.120415    
## poly(duration, 2)2:monthmar         -1.087e+01  4.397e+01  -0.247 0.804745    
## poly(duration, 2)1:monthmay          1.232e+02  1.285e+01   9.585  < 2e-16 ***
## poly(duration, 2)2:monthmay         -9.163e+01  1.990e+01  -4.605 4.12e-06 ***
## poly(duration, 2)1:monthnov          3.355e+01  1.417e+01   2.367 0.017912 *  
## poly(duration, 2)2:monthnov         -2.517e+00  2.193e+01  -0.115 0.908616    
## poly(duration, 2)1:monthoct         -2.932e+01  2.335e+01  -1.256 0.209234    
## poly(duration, 2)2:monthoct         -1.691e+01  3.077e+01  -0.550 0.582656    
## poly(duration, 2)1:monthsep          1.902e+01  2.562e+01   0.742 0.457805    
## poly(duration, 2)2:monthsep         -1.448e+01  3.733e+01  -0.388 0.698092    
## poly(duration, 2)1:housingyes        3.936e+01  7.613e+00   5.170 2.34e-07 ***
## poly(duration, 2)2:housingyes        6.839e+00  1.122e+01   0.610 0.542003    
## poly(duration, 2)1:jobblue-collar    1.479e+01  1.270e+01   1.164 0.244446    
## poly(duration, 2)2:jobblue-collar   -3.740e+01  1.759e+01  -2.126 0.033508 *  
## poly(duration, 2)1:jobentrepreneur   3.039e+01  2.225e+01   1.366 0.172054    
## poly(duration, 2)2:jobentrepreneur  -2.235e+01  2.567e+01  -0.871 0.383903    
## poly(duration, 2)1:jobhousemaid     -3.134e+01  2.017e+01  -1.554 0.120162    
## poly(duration, 2)2:jobhousemaid      2.372e+00  2.666e+01   0.089 0.929120    
## poly(duration, 2)1:jobmanagement    -1.748e+01  1.182e+01  -1.479 0.139084    
## poly(duration, 2)2:jobmanagement    -1.384e+01  1.592e+01  -0.869 0.384736    
## poly(duration, 2)1:jobretired       -3.606e+01  1.624e+01  -2.221 0.026365 *  
## poly(duration, 2)2:jobretired       -4.471e+01  2.374e+01  -1.883 0.059694 .  
## poly(duration, 2)1:jobself-employed  1.699e+01  1.995e+01   0.852 0.394440    
## poly(duration, 2)2:jobself-employed  2.287e+01  3.022e+01   0.757 0.449178    
## poly(duration, 2)1:jobservices       4.084e-01  1.483e+01   0.028 0.978025    
## poly(duration, 2)2:jobservices      -3.233e+00  2.193e+01  -0.147 0.882805    
## poly(duration, 2)1:jobstudent       -4.411e+01  2.767e+01  -1.594 0.110832    
## poly(duration, 2)2:jobstudent       -9.919e+01  4.644e+01  -2.136 0.032687 *  
## poly(duration, 2)1:jobtechnician    -9.709e+00  1.255e+01  -0.774 0.439194    
## poly(duration, 2)2:jobtechnician    -2.804e+01  1.626e+01  -1.725 0.084578 .  
## poly(duration, 2)1:jobunemployed    -1.334e+01  1.860e+01  -0.717 0.473499    
## poly(duration, 2)2:jobunemployed    -4.697e+01  2.503e+01  -1.877 0.060521 .  
## poly(duration, 2)1:jobunknown        8.459e+01  7.653e+01   1.105 0.269025    
## poly(duration, 2)2:jobunknown        5.402e+01  1.390e+02   0.389 0.697573    
## monthaug:balance                     1.143e-05  2.540e-05   0.450 0.652622    
## monthdec:balance                    -2.177e-06  6.141e-05  -0.035 0.971728    
## monthfeb:balance                     9.858e-06  3.060e-05   0.322 0.747352    
## monthjan:balance                     1.554e-04  5.057e-05   3.073 0.002122 ** 
## monthjul:balance                     1.535e-05  2.966e-05   0.518 0.604707    
## monthjun:balance                    -2.463e-05  2.734e-05  -0.901 0.367574    
## monthmar:balance                    -8.173e-05  4.772e-05  -1.713 0.086744 .  
## monthmay:balance                    -1.180e-05  2.751e-05  -0.429 0.668025    
## monthnov:balance                    -1.718e-05  2.648e-05  -0.649 0.516372    
## monthoct:balance                    -4.975e-05  3.210e-05  -1.550 0.121260    
## monthsep:balance                    -5.463e-05  4.506e-05  -1.212 0.225342    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 22845  on 31648  degrees of freedom
## Residual deviance: 14013  on 31546  degrees of freedom
## AIC: 14219
## 
## Number of Fisher Scoring iterations: 7

# Generate an ROC curve to help us determine the best threshold
logit_complex_predictions = predict(logit_complex, test, type = "response")
logit_complex_roc = roc(response = test$y, predictor = logit_complex_predictions, levels = c("no", "yes"))

## Setting direction: controls < cases

auc(logit_complex_roc)

## Area under the curve: 0.9145

plot(logit_complex_roc, print.thres = "best", col = "black")

# Predict
threshold = 0.124
logit_complex_predictions_categorical = factor(ifelse(predict(logit_complex, test, "response") >= threshold, "yes", "no"), levels = c("no", "yes"))
confusionMatrix(data = logit_complex_predictions_categorical, reference = factor(test$y, levels = c("no", "yes")))

## Confusion Matrix and Statistics
## 
##           Reference
## Prediction    no   yes
##        no  10033   242
##        yes  1943  1344
##                                          
##                Accuracy : 0.8389         
##                  95% CI : (0.8326, 0.845)
##     No Information Rate : 0.8831         
##     P-Value [Acc > NIR] : 1              
##                                          
##                   Kappa : 0.4676         
##                                          
##  Mcnemar's Test P-Value : <2e-16         
##                                          
##             Sensitivity : 0.8378         
##             Specificity : 0.8474         
##          Pos Pred Value : 0.9764         
##          Neg Pred Value : 0.4089         
##              Prevalence : 0.8831         
##          Detection Rate : 0.7398         
##    Detection Prevalence : 0.7576         
##       Balanced Accuracy : 0.8426         
##                                          
##        'Positive' Class : no             
##

AIC(logit_complex)

## [1] 14218.68

anova(logit_complex, test="Chisq")

## Analysis of Deviance Table
## 
## Model: binomial, link: logit
## 
## Response: factor(y)
## 
## Terms added sequentially (first to last)
## 
## 
##                            Df Deviance Resid. Df Resid. Dev  Pr(>Chi)    
## NULL                                       31648      22845              
## poutcome                    3   1952.1     31645      20893 < 2.2e-16 ***
## poly(duration, 2)           2   4185.8     31643      16707 < 2.2e-16 ***
## month                      11   1153.3     31632      15554 < 2.2e-16 ***
## contact                     2    441.3     31630      15112 < 2.2e-16 ***
## housing                     1    308.6     31629      14804 < 2.2e-16 ***
## job                        11    140.5     31618      14663 < 2.2e-16 ***
## campaign                    1     53.6     31617      14610 2.425e-13 ***
## loan                        1     46.6     31616      14563 8.685e-12 ***
## marital                     2     38.1     31614      14525 5.439e-09 ***
## education                   3     19.9     31611      14505 0.0001801 ***
## day                         1      8.0     31610      14497 0.0047028 ** 
## balance                     1      6.2     31609      14491 0.0124291 *  
## poutcome:poly(duration, 2)  6     87.4     31603      14404 < 2.2e-16 ***
## poly(duration, 2):month    22    280.9     31581      14123 < 2.2e-16 ***
## poly(duration, 2):housing   2     41.8     31579      14081 8.452e-10 ***
## poly(duration, 2):job      22     44.2     31557      14037 0.0033831 ** 
## month:balance              11     24.0     31546      14013 0.0129425 *  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

2. LDA/QDA

training %>% ggplot(aes(x =age,y = balance, color = as.factor(y))) + 
  geom_point()+geom_density_2d()

training %>% ggplot(aes(x =age,y = day, color = as.factor(y))) + 
  geom_point()+geom_density_2d()

training %>% ggplot(aes(x =age,y = duration, color = as.factor(y))) + 
  geom_point()+geom_density_2d()

training %>% ggplot(aes(x =age,y = campaign, color = as.factor(y))) + 
  geom_point()+geom_density_2d()

training %>% ggplot(aes(x =log(balance),y = day, color = as.factor(y))) + 
  geom_point()+geom_density_2d()

## Warning in log(balance): NaNs produced

## Warning in log(balance): NaNs produced

## Warning in log(balance): NaNs produced

## Warning: Removed 5189 rows containing non-finite values (stat_density2d).

## Warning: Removed 2714 rows containing missing values (geom_point).

training %>% ggplot(aes(x =log(balance),y = duration, color = as.factor(y))) + 
  geom_point()+geom_density_2d()

## Warning in log(balance): NaNs produced

## Warning in log(balance): NaNs produced

## Warning in log(balance): NaNs produced

## Warning: Removed 5189 rows containing non-finite values (stat_density2d).

## Warning: Removed 2714 rows containing missing values (geom_point).

training %>% ggplot(aes(x =log(balance),y = log(campaign), color = as.factor(y))) + 
  geom_point()+geom_density_2d()

## Warning in log(balance): NaNs produced

## Warning in log(balance): NaNs produced

## Warning in log(balance): NaNs produced

## Warning: Removed 5189 rows containing non-finite values (stat_density2d).

## Warning: Removed 2714 rows containing missing values (geom_point).

training %>% ggplot(aes(x =day,y = duration, color = as.factor(y))) + 
  geom_point()+geom_density_2d()

training %>% ggplot(aes(x =day,y = campaign, color = as.factor(y))) + 
  geom_point()+geom_density_2d()

#### LDA

AUROC: 0.9117 Accuracy: 0.826
Sensitivity: 0.8217
Specificity: 0.8581
Prevalence: 0.8831 PPV: 0.9776
NPV: 0.3893

set.seed(1234)
fitControl = trainControl(method = "repeatedcv", number = 5, repeats = 1, classProbs = TRUE, summaryFunction = mnLogLoss)
lda_base = train(factor(y) ~ ., data = training, method = "lda", trControl = fitControl, metric = "logLoss")
summary(lda_base)

##             Length Class      Mode     
## prior        2     -none-     numeric  
## counts       2     -none-     numeric  
## means       84     -none-     numeric  
## scaling     42     -none-     numeric  
## lev          2     -none-     character
## svd          1     -none-     numeric  
## N            1     -none-     numeric  
## call         3     -none-     call     
## xNames      42     -none-     character
## problemType  1     -none-     character
## tuneValue    1     data.frame list     
## obsLevels    2     -none-     character
## param        0     -none-     list

# Generate an ROC curve to help us determine the best threshold
lda_base_predictions = predict(lda_base, test, type = "prob")$yes
lda_base_roc = roc(response = test$y, predictor = lda_base_predictions, levels = c("no", "yes"))

## Setting direction: controls < cases

auc(lda_base_roc)

## Area under the curve: 0.9117

plot(lda_base_roc, print.thres = "best", col = "black")

# Predict
threshold = 0.045
lda_base_predictions_categorical = factor(ifelse(predict(lda_base, test, type = "prob")$yes >= threshold, "yes", "no"), levels = c("no", "yes"))
confusionMatrix(data = lda_base_predictions_categorical, reference = factor(test$y, levels = c("no", "yes")))

## Confusion Matrix and Statistics
## 
##           Reference
## Prediction   no  yes
##        no  9841  225
##        yes 2135 1361
##                                           
##                Accuracy : 0.826           
##                  95% CI : (0.8195, 0.8323)
##     No Information Rate : 0.8831          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0.4466          
##                                           
##  Mcnemar's Test P-Value : <2e-16          
##                                           
##             Sensitivity : 0.8217          
##             Specificity : 0.8581          
##          Pos Pred Value : 0.9776          
##          Neg Pred Value : 0.3893          
##              Prevalence : 0.8831          
##          Detection Rate : 0.7256          
##    Detection Prevalence : 0.7422          
##       Balanced Accuracy : 0.8399          
##                                           
##        'Positive' Class : no              
##

QDA

NOTE: QDA does not perform very well here. The optimal threshold was 0 which returns NaN’s in certain performance metrics

AUROC: 0.838 Accuracy: 0.1169
Sensitivity: 0.0
Specificity: 1.0
Prevalence: 0.8831 PPV: NaN
NPV: 0.1169

set.seed(1234)
fitControl = trainControl(method = "repeatedcv", number = 5, repeats = 1, classProbs = TRUE, summaryFunction = mnLogLoss)
qda_base = train(factor(y) ~ ., data = training, method = "qda", trControl = fitControl, metric = "logLoss")
summary(qda_base)

##             Length Class      Mode     
## prior          2   -none-     numeric  
## counts         2   -none-     numeric  
## means         84   -none-     numeric  
## scaling     3528   -none-     numeric  
## ldet           2   -none-     numeric  
## lev            2   -none-     character
## N              1   -none-     numeric  
## call           3   -none-     call     
## xNames        42   -none-     character
## problemType    1   -none-     character
## tuneValue      1   data.frame list     
## obsLevels      2   -none-     character
## param          0   -none-     list

# Generate an ROC curve to help us determine the best threshold
qda_base_predictions = predict(qda_base, test, type = "prob")$yes
qda_base_roc = roc(response = test$y, predictor = qda_base_predictions, levels = c("no", "yes"))

## Setting direction: controls < cases

auc(qda_base_roc)

## Area under the curve: 0.838

plot(qda_base_roc, print.thres = "best", col = "black")

# Predict
threshold = 0
qda_base_predictions_categorical = factor(ifelse(predict(qda_base, test, type = "prob")$yes >= threshold, "yes", "no"), levels = c("no", "yes"))
confusionMatrix(data = qda_base_predictions_categorical, reference = factor(test$y, levels = c("no", "yes")))

## Confusion Matrix and Statistics
## 
##           Reference
## Prediction    no   yes
##        no      0     0
##        yes 11976  1586
##                                           
##                Accuracy : 0.1169          
##                  95% CI : (0.1116, 0.1225)
##     No Information Rate : 0.8831          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0               
##                                           
##  Mcnemar's Test P-Value : <2e-16          
##                                           
##             Sensitivity : 0.0000          
##             Specificity : 1.0000          
##          Pos Pred Value :    NaN          
##          Neg Pred Value : 0.1169          
##              Prevalence : 0.8831          
##          Detection Rate : 0.0000          
##    Detection Prevalence : 0.0000          
##       Balanced Accuracy : 0.5000          
##                                           
##        'Positive' Class : no              
##

3. Non-parametric model

KNN Model

AUROC: 0.7963 Accuracy: 0.8084
Sensitivity: 0.8286
Specificity: 0.6564
Prevalence: 0.8831 PPV: 0.9479
NPV: 0.3365

## KNN 
set.seed(1234)
fitControl = trainControl(method = "repeatedcv", number = 5, repeats = 1, classProbs = TRUE, summaryFunction = mnLogLoss)
knn.base = train(factor(y) ~ ., data = training, method = "knn", trControl = fitControl, metric = "logLoss")

# Generate an ROC curve to help us determine the best threshold
knn.base_predictions = predict(knn.base, test, type = "prob")$yes
knn.base_roc = roc(response = test$y, predictor = knn.base_predictions, levels = c("no", "yes"))

## Setting direction: controls < cases

auc(knn.base_roc)

## Area under the curve: 0.7963

plot(knn.base_roc, print.thres = "best", col = "black")

# Predict
threshold = 0.211
knn.base_predictions_categorical = factor(ifelse(predict(knn.base, test, type = "prob")$yes >= threshold, "yes", "no"), levels = c("no", "yes"))
confusionMatrix(data = knn.base_predictions_categorical, reference = factor(test$y, levels = c("no", "yes")))

## Confusion Matrix and Statistics
## 
##           Reference
## Prediction   no  yes
##        no  9923  545
##        yes 2053 1041
##                                          
##                Accuracy : 0.8084         
##                  95% CI : (0.8017, 0.815)
##     No Information Rate : 0.8831         
##     P-Value [Acc > NIR] : 1              
##                                          
##                   Kappa : 0.3433         
##                                          
##  Mcnemar's Test P-Value : <2e-16         
##                                          
##             Sensitivity : 0.8286         
##             Specificity : 0.6564         
##          Pos Pred Value : 0.9479         
##          Neg Pred Value : 0.3365         
##              Prevalence : 0.8831         
##          Detection Rate : 0.7317         
##    Detection Prevalence : 0.7719         
##       Balanced Accuracy : 0.7425         
##                                          
##        'Positive' Class : no             
##

Random Forest Model

AUROC: 0.9281 Accuracy: 0.8363
Sensitivity: 0.8267
Specificity: 0.9086
Prevalence: 0.8831 PPV: 0.9856
NPV: 0.4098

## Random Forest
set.seed(1234)
fitControl = trainControl(method = "repeatedcv", number = 5, repeats = 1, classProbs = TRUE, summaryFunction = mnLogLoss)
rf.base = train(y ~ ., data = training, method = "rf", trControl = fitControl, metric = "logLoss")
summary(rf.base)

##                 Length Class      Mode     
## call                4  -none-     call     
## type                1  -none-     character
## predicted       31649  factor     numeric  
## err.rate         1500  -none-     numeric  
## confusion           6  -none-     numeric  
## votes           63298  matrix     numeric  
## oob.times       31649  -none-     numeric  
## classes             2  -none-     character
## importance         42  -none-     numeric  
## importanceSD        0  -none-     NULL     
## localImportance     0  -none-     NULL     
## proximity           0  -none-     NULL     
## ntree               1  -none-     numeric  
## mtry                1  -none-     numeric  
## forest             14  -none-     list     
## y               31649  factor     numeric  
## test                0  -none-     NULL     
## inbag               0  -none-     NULL     
## xNames             42  -none-     character
## problemType         1  -none-     character
## tuneValue           1  data.frame list     
## obsLevels           2  -none-     character
## param               0  -none-     list

# Generate an ROC curve to help us determine the best threshold
rf.base_predictions = predict(rf.base, test, type = "prob")$yes
rf.base_roc = roc(response = test$y, predictor = rf.base_predictions, levels = c("no", "yes"))

## Setting direction: controls < cases

auc(rf.base_roc)

## Area under the curve: 0.9281

plot(rf.base_roc, print.thres = "best", col = "black")

# Predict
threshold = 0.123
rf.base_predictions_categorical = factor(ifelse(predict(rf.base, test, type = "prob")$yes >= threshold, "yes", "no"), levels = c("no", "yes"))
confusionMatrix(data = rf.base_predictions_categorical, reference = factor(test$y, levels = c("no", "yes")))

## Confusion Matrix and Statistics
## 
##           Reference
## Prediction   no  yes
##        no  9901  145
##        yes 2075 1441
##                                         
##                Accuracy : 0.8363        
##                  95% CI : (0.83, 0.8425)
##     No Information Rate : 0.8831        
##     P-Value [Acc > NIR] : 1             
##                                         
##                   Kappa : 0.4813        
##                                         
##  Mcnemar's Test P-Value : <2e-16        
##                                         
##             Sensitivity : 0.8267        
##             Specificity : 0.9086        
##          Pos Pred Value : 0.9856        
##          Neg Pred Value : 0.4098        
##              Prevalence : 0.8831        
##          Detection Rate : 0.7301        
##    Detection Prevalence : 0.7407        
##       Balanced Accuracy : 0.8677        
##                                         
##        'Positive' Class : no            
##

eXtreme Gradient Boosting

AUROC: 0.9333
Accuracy: 0.834
Sensitivity: 0.8228
Specificity: 0.9187
Prevalence: 0.8831 PPV: 0.9871
NPV: 0.4071

## XGB
set.seed(1234)
fitControl = trainControl(method = "repeatedcv", number = 5, repeats = 1, classProbs = TRUE, summaryFunction = mnLogLoss)
xgb.base = train(factor(y) ~ ., data = training, method = "xgbTree", trControl = fitControl, metric = "logLoss")

## [13:17:52] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:52] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:52] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:52] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:53] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:53] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:53] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:53] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:54] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:54] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:54] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:54] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:55] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:55] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:55] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:55] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:57] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:57] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:57] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:57] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:58] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:58] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:58] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:17:58] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:00] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:00] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:00] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:00] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:10] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:10] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:10] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:10] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:12] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:12] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:12] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:12] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:14] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:14] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:14] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:14] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:17] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:17] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:17] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:17] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:19] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:19] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:19] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:19] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:21] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:21] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:21] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:21] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:22] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:22] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:22] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:22] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:24] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:24] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:24] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:24] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:26] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:26] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:26] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:26] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:27] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:27] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:27] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:27] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:28] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:28] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:28] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:28] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:30] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:30] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:30] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:30] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:33] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:33] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:33] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:33] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:35] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:36] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:36] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:36] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:37] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:37] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:37] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:37] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:46] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:46] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:46] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:46] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:47] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:47] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:47] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:47] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:48] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:48] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:48] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:48] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:50] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:50] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:50] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:50] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:51] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:51] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:51] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:51] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:52] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:52] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:52] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:52] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:53] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:53] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:53] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:53] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:54] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:54] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:55] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:55] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:58] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:58] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:58] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:58] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:18:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:05] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:05] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:05] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:05] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:11] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:11] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:11] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:11] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:13] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:13] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:13] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:13] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:14] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:14] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:14] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:14] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:16] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:16] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:16] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:16] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:17] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:17] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:17] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:17] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:19] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:19] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:19] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:19] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:22] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:22] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:22] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:22] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:25] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:25] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:25] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:25] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:26] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:26] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:26] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:26] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:28] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:28] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:28] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:28] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:30] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:30] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:30] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:30] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:34] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:34] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:34] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:34] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:36] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:36] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:36] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:36] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:41] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:41] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:41] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:41] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:43] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:43] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:43] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:43] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:45] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:45] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:45] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:45] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:46] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:46] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:46] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:46] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:48] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:48] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:48] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:48] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:50] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:50] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:50] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:50] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:52] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:52] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:52] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:52] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:54] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:54] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:54] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:54] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:55] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:55] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:55] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:55] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:57] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:57] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:57] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:57] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:19:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:10] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:10] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:10] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:10] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:11] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:11] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:11] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:11] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:12] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:12] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:12] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:12] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:13] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:13] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:13] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:13] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:14] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:14] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:14] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:14] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:17] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:17] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:17] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:17] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:21] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:21] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:21] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:21] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:25] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:25] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:25] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:25] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:27] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:27] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:27] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:27] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:29] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:29] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:29] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:29] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:34] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:34] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:34] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:34] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:36] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:36] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:36] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:36] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:37] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:37] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:37] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:37] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:38] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:38] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:38] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:38] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:40] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:40] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:40] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:40] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:41] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:41] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:41] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:41] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:43] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:43] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:43] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:43] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:46] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:46] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:46] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:46] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:47] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:47] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:47] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:47] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:51] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:51] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:51] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:51] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:53] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:53] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:53] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:53] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:55] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:55] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:55] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:55] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:20:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:05] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:05] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:05] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:05] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:07] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:07] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:07] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:07] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:10] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:10] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:10] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:10] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:12] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:12] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:12] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:12] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:13] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:13] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:13] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:13] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:16] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:16] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:16] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:16] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:20] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:22] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:22] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:22] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:22] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:26] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:26] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:26] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:26] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:28] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:28] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:28] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:28] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:30] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:30] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:30] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:30] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:31] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:31] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:31] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:31] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:32] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:33] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:33] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:33] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:33] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:34] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:34] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:34] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:34] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:35] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:35] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:35] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:35] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:36] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:36] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:36] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:36] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:38] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:38] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:38] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:38] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:39] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:40] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:40] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:40] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:40] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:42] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:44] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:45] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:45] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:45] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:45] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:47] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:47] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:47] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:47] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:49] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:51] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:51] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:51] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:51] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:54] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:54] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:54] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:54] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:56] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:58] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:58] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:58] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:58] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:21:59] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:00] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:00] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:00] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:00] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:01] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:02] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:02] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:02] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:02] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:03] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:04] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:05] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:05] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:05] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:05] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:06] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:08] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:09] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:11] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:11] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:11] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:11] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:12] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:12] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:13] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:13] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:15] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:16] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:16] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:16] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:16] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:18] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:21] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:21] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:21] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:21] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:23] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:25] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:25] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:25] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.
## [13:22:25] WARNING: src/c_api/c_api.cc:935: `ntree_limit` is deprecated, use `iteration_range` instead.

summary(xgb.base)

##               Length Class              Mode       
## handle             1 xgb.Booster.handle externalptr
## raw           175792 -none-             raw        
## niter              1 -none-             numeric    
## call               5 -none-             call       
## params             8 -none-             list       
## callbacks          1 -none-             list       
## feature_names     42 -none-             character  
## nfeatures          1 -none-             numeric    
## xNames            42 -none-             character  
## problemType        1 -none-             character  
## tuneValue          7 data.frame         list       
## obsLevels          2 -none-             character  
## param              0 -none-             list

# Generate an ROC curve to help us determine the best threshold
xgb.base_predictions = predict(xgb.base, test, type = "prob")$yes
xgb.base_roc = roc(response = test$y, predictor = xgb.base_predictions, levels = c("no", "yes"))
auc(xgb.base_roc)

## Area under the curve: 0.9324

plot(xgb.base_roc, print.thres = "best", col = "black")

# Predict
threshold = 0.096
xgb.base_predictions_categorical = factor(ifelse(predict(xgb.base, test, type = "prob")$yes >= threshold, "yes", "no"), levels = c("no", "yes"))
confusionMatrix(data = xgb.base_predictions_categorical, reference = factor(test$y, levels = c("no", "yes")))

## Confusion Matrix and Statistics
## 
##           Reference
## Prediction   no  yes
##        no  9854  129
##        yes 2122 1457
##                                           
##                Accuracy : 0.834           
##                  95% CI : (0.8277, 0.8402)
##     No Information Rate : 0.8831          
##     P-Value [Acc > NIR] : 1               
##                                           
##                   Kappa : 0.4799          
##                                           
##  Mcnemar's Test P-Value : <2e-16          
##                                           
##             Sensitivity : 0.8228          
##             Specificity : 0.9187          
##          Pos Pred Value : 0.9871          
##          Neg Pred Value : 0.4071          
##              Prevalence : 0.8831          
##          Detection Rate : 0.7266          
##    Detection Prevalence : 0.7361          
##       Balanced Accuracy : 0.8707          
##                                           
##        'Positive' Class : no              
##

Check All Models AIC

AIC(model1)

## [1] 15206.49

AIC(adj.model1)

## [1] 15202.89

AIC(pen.model1)

## [1] 15201.86

AIC(fit.step)

## [1] 15201.86

AIC(logit_step)

## [1] 15484.51

AIC(logit_complex)

## [1] 14218.68