initial commit

Author: David Foster

Diff for: .DS_Store

@@ -0,0 +1,2 @@
+^.*\.Rproj$
+^\.Rproj\.user$

Diff for: .Rbuildignore

@@ -0,0 +1,11 @@
+Package: xgboostExplainer
+Title: XGBoost Model Explainer
+Version: 0.1
+Authors@R: person("David", "Foster", email = "[email protected]", role = c("aut", "cre"))
+Description: XGBoost is a very successful machine learning package based on boosted trees. This package allows the predictions from an xgboost model to be split into the impact of each feature, making the model as transparent as a linear regression or decision tree.
+Depends: R (>= 3.4.1)
+Imports: data.table, xgboost, waterfalls, scales, ggplot2
+License: GPL-3
+Encoding: UTF-8
+LazyData: true
+RoxygenNote: 6.0.1.9000

Diff for: DESCRIPTION

@@ -0,0 +1,10 @@
+# Generated by roxygen2: do not edit by hand
+
+export(buildExplainer)
+export(explainPredictions)
+export(showWaterfall)
+import(data.table)
+import(ggplot2)
+import(scales)
+import(waterfalls)
+import(xgboost)

Diff for: NAMESPACE

@@ -0,0 +1,67 @@
+#' Step 1: Build an xgboostExplainer
+#'
+#' This function outputs an xgboostExplainer (a data table that stores the feature impact breakdown for each leaf of each tree in an xgboost model). It is required as input into the explainPredictions and showWaterfall functions.
+#' @param xgb.model A trained xgboost model
+#' @param trainingData A DMatrix of data used to train the model
+#' @param type The objective function of the model - either "binary" (for binary:logistic) or "regression" (for reg:linear)
+#' @param base_score Default 0.5. The base_score variable of the xgboost model.
+#' @return The XGBoost Explainer for the model. This is a data table where each row is a leaf of a tree in the xgboost model
+#'  and each column is the impact of each feature on the prediction at the leaf.
+#'  
+#'  The leaf and tree columns uniquely identify the node.
+#'  
+#'  The sum of the other columns equals the prediction at the leaf (log-odds if binary response).
+#'  
+#'  The 'intercept' column is identical for all rows and is analogous to the intercept term in a linear / logistic regression.
+#' 
+#' @export
+#' @import data.table
+#' @import xgboost
+#' @examples
+#' library(xgboost)
+#' library(xgboostExplainer)
+#' 
+#' set.seed(123)
+#' 
+#' data(agaricus.train, package='xgboost')
+#' 
+#' X = as.matrix(agaricus.train$data)
+#' y = agaricus.train$label
+#' 
+#' train_idx = 1:5000
+#' 
+#' xgb.train.data <- xgb.DMatrix(X[train_idx,], label = y[train_idx])
+#' xgb.test.data <- xgb.DMatrix(X[-train_idx,])
+#' 
+#' param <- list(objective = "binary:logistic")
+#' xgb.model <- xgboost(param =param,  data = xgb.train.data, nrounds=3)
+#' 
+#' col_names = colnames(X)
+#' 
+#' pred.train = predict(xgb.model,X)
+#' nodes.train = predict(xgb.model,X,predleaf =TRUE)
+#' trees = xgb.model.dt.tree(col_names, model = xgb.model)
+#' 
+#' #### The XGBoost Explainer
+#' explainer = buildExplainer(xgb.model,xgb.train.data, type="binary", base_score = 0.5)
+#' pred.breakdown = explainPredictions(xgb.model, explainer, xgb.test.data)
+#' 
+#' showWaterfall(xgb.model, explainer, slice(xgb.test.data, as.integer(2)), type = "binary")
+#' showWaterfall(xgb.model, explainer, slice(xgb.test.data, as.integer(8)), type = "binary")
+
+
+buildExplainer = function(xgb.model, trainingData, type = "binary", base_score = 0.5){
+  
+  col_names = attr(trainingData, ".Dimnames")[[2]]
+  trees = xgb.model.dt.tree(col_names, model = xgb.model, n_first_tree = xgb.model$best_ntreelimit - 1)
+  nodes.train = predict(xgb.model,xgb.train.data,predleaf =TRUE)
+  
+  cat('\nSTEP 1 of 2')
+  tree_list = getStatsForTrees(trees, nodes.train, type = type, base_score = base_score)
+  cat('\n\nSTEP 2 of 2')
+  explainer = buildExplainerFromTreeList(tree_list,col_names)
+  
+  cat('\n\nDONE!\n')
+  
+  return (explainer)
+}

Diff for: R/.DS_Store

@@ -0,0 +1,28 @@
+
+#' @import data.table
+#' @import xgboost
+
+buildExplainerFromTreeList = function(tree_list,col_names){
+  
+  ####accepts a list of trees and column names
+  ####outputs a data table, of the impact of each variable + intercept, for each leaf
+
+  tree_list_breakdown <- vector("list", length(col_names)  + 3)
+  names(tree_list_breakdown) = c(col_names,'intercept', 'leaf','tree')
+
+  num_trees = length(tree_list)
+  
+  cat('\n\nGetting breakdown for each leaf of each tree...\n')
+  pb <- txtProgressBar(style=3)
+  
+  for (x in 1:num_trees){
+    tree = tree_list[[x]]
+    tree_breakdown = getTreeBreakdown(tree, col_names)
+    tree_breakdown$tree = x - 1
+    tree_list_breakdown = rbindlist(append(list(tree_list_breakdown),list(tree_breakdown)))
+    setTxtProgressBar(pb, x / num_trees)
+  }
+  
+  return (tree_list_breakdown)
+  
+}

Diff for: R/buildExplainer.R

@@ -0,0 +1,76 @@
+#' Step 2: Get multiple prediction breakdowns from a trained xgboost model
+#'
+#' This function outputs the feature impact breakdown of a set of predictions made using an xgboost model.
+#' @param xgb.model A trained xgboost model
+#' @param explainer The output from the buildExplainer function, for this model
+#' @param data A DMatrix of data to be explained
+#' @return A data table where each row is an observation in the data and each column is the impact of each feature on the prediction.
+#' 
+#' The sum of the row equals the prediction of the xgboost model for this observation (log-odds if binary response). 
+#' 
+#' @export
+#' @import data.table
+#' @import xgboost
+#' @examples
+#' library(xgboost)
+#' library(xgboostExplainer)
+#' 
+#' set.seed(123)
+#' 
+#' data(agaricus.train, package='xgboost')
+#' 
+#' X = as.matrix(agaricus.train$data)
+#' y = agaricus.train$label
+#' 
+#' train_idx = 1:5000
+#' 
+#' xgb.train.data <- xgb.DMatrix(X[train_idx,], label = y[train_idx])
+#' xgb.test.data <- xgb.DMatrix(X[-train_idx,])
+#' 
+#' param <- list(objective = "binary:logistic")
+#' xgb.model <- xgboost(param =param,  data = xgb.train.data, nrounds=3)
+#' 
+#' col_names = colnames(X)
+#' 
+#' pred.train = predict(xgb.model,X)
+#' nodes.train = predict(xgb.model,X,predleaf =TRUE)
+#' trees = xgb.model.dt.tree(col_names, model = xgb.model)
+#' 
+#' #### The XGBoost Explainer
+#' explainer = buildExplainer(xgb.model,xgb.train.data, type="binary", base_score = 0.5)
+#' pred.breakdown = explainPredictions(xgb.model, explainer, xgb.test.data)
+#' 
+#' showWaterfall(xgb.model, explainer, slice(xgb.test.data, as.integer(2)), type = "binary")
+#' showWaterfall(xgb.model, explainer, slice(xgb.test.data, as.integer(8)), type = "binary")
+
+explainPredictions = function(xgb.model, explainer ,data){
+  
+  #Accepts data table of the breakdown for each leaf of each tree and the node matrix
+  #Returns the breakdown for each prediction as a data table
+  
+  nodes = predict(xgb.model,data,predleaf =TRUE)
+  
+  colnames = names(explainer)[1:(ncol(explainer)-2)]
+  
+  preds_breakdown = data.table(matrix(0,nrow = nrow(nodes), ncol = length(colnames)))
+  setnames(preds_breakdown, colnames)
+  
+  num_trees = ncol(nodes)
+  
+  cat('\n\nExtracting the breakdown of each prediction...\n')
+  pb <- txtProgressBar(style=3)
+  for (x in 1:num_trees){
+    nodes_for_tree = nodes[,x]
+    tree_breakdown = explainer[tree==x-1]
+    
+    preds_breakdown_for_tree = tree_breakdown[match(nodes_for_tree, tree_breakdown$leaf),]
+    preds_breakdown = preds_breakdown + preds_breakdown_for_tree[,colnames,with=FALSE]
+    
+    setTxtProgressBar(pb, x / num_trees)
+  }
+  
+  cat('\n\nDONE!\n')
+
+  return (preds_breakdown)
+  
+}

Diff for: R/buildExplainerFromTreeList.R

@@ -0,0 +1,17 @@
+
+#' @import data.table
+#' @import xgboost
+findLeaves = function(tree, currentnode){
+  
+  if (tree[currentnode,'Feature']=='Leaf'){
+    leaves = currentnode
+  }else{
+    leftnode = tree[currentnode,Yes]
+    rightnode = tree[currentnode,No]
+    leaves = c(findLeaves(tree,'leftnode',with=FALSE),findLeaves(tree,'rightnode',with=FALSE))
+  }
+  
+  return (sort(leaves))
+  
+  
+}

Diff for: R/explainPredictions.R

@@ -0,0 +1,17 @@
+
+#' @import data.table
+#' @import xgboost
+findPath = function(tree, currentnode, path = c()){
+  
+  #accepts a tree data table, and the node to reach
+  #path is used in the recursive function - do not set this
+  
+  while(currentnode>0){
+    path = c(path,currentnode)
+    currentlabel = tree[Node==currentnode,ID]
+    currentnode = c(tree[Yes==currentlabel,Node],tree[No==currentlabel,Node])
+  }
+  return (sort(c(path,0)))
+  
+}
+

Diff for: R/findLeaves.R

@@ -0,0 +1,25 @@
+
+#' @import data.table
+#' @import xgboost
+getLeafBreakdown = function(tree,leaf,col_names){
+  
+  ####accepts a tree, the leaf id to breakdown and column names
+  ####outputs a list of the impact of each variable + intercept
+  
+  impacts = as.list(rep(0,length(col_names)))
+  names(impacts) = col_names
+  
+  path = findPath(tree,leaf)
+  reduced_tree = tree[Node %in% path,.(Feature,uplift_weight)]
+  
+  impacts$intercept=reduced_tree[1,uplift_weight]
+  reduced_tree[,uplift_weight:=shift(uplift_weight,type='lead')]
+  
+  tmp = reduced_tree[,.(sum=sum(uplift_weight)),by=Feature]
+  tmp = tmp[-nrow(tmp)]
+  impacts[tmp[,Feature]]=tmp[,sum]
+  
+  return (impacts)
+}
+
+

Diff for: R/findPath.R

@@ -0,0 +1,70 @@
+
+#' @import data.table
+#' @import xgboost
+getStatsForTrees = function(trees, nodes.train, type = "binary", base_score = 0.5){
+  #Accepts data table of tree (the output of xgb.model.dt.tree)
+  #Returns a list of tree, with the stats filled in
+  
+  tree_list = copy(trees)
+  tree_list[,leaf := Feature == 'Leaf']
+  tree_list[,H:=Cover]
+  
+  non.leaves = which(tree_list[,leaf]==F)
+
+  
+  # The default cover (H) seems to lose precision so this loop recalculates it for each node of each tree
+  cat('\n\nRecalculating the cover for each non-leaf... \n')
+  pb <- txtProgressBar(style=3)
+  j = 0
+  for (i in rev(non.leaves)){
+    left = tree_list[i,Yes]
+    right = tree_list[i,No]
+    tree_list[i,H:=tree_list[ID==left,H] + tree_list[ID==right,H]]
+    j=j+1
+    setTxtProgressBar(pb, j / length(non.leaves))
+  }
+  
+
+  if (type == 'regression'){
+    base_weight = base_score
+  } else{
+    base_weight = log(base_score / (1-base_score))
+  }
+  
+  tree_list[leaf==T,weight:=base_weight + Quality]
+  
+  tree_list[,previous_weight:=base_weight]
+  tree_list[1,previous_weight:=0]
+  
+  tree_list[leaf==T,G:=-weight*H]
+  
+  tree_list = split(tree_list,as.factor(tree_list$Tree))
+  num_tree_list = length(tree_list)
+  treenums =  as.character(0:(num_tree_list-1))
+  t = 0
+  cat('\n\nFinding the stats for the xgboost trees...\n')
+  pb <- txtProgressBar(style=3)
+  for (tree in tree_list){
+    t=t+1
+    num_nodes = nrow(tree)
+    non_leaf_rows = rev(which(tree[,leaf]==F))
+    for (r in non_leaf_rows){
+        left = tree[r,Yes]
+        right = tree[r,No]
+        leftG = tree[ID==left,G]
+        rightG = tree[ID==right,G]
+        
+        tree[r,G:=leftG+rightG]
+        w=tree[r,-G/H]
+        
+        tree[r,weight:=w]
+        tree[ID==left,previous_weight:=w]
+        tree[ID==right,previous_weight:=w]
+    }
+    
+    tree[,uplift_weight:=weight-previous_weight]
+    setTxtProgressBar(pb, t / num_tree_list)
+  }
+  
+  return (tree_list)
+}

Diff for: R/getLeafBreakdown.R

@@ -0,0 +1,24 @@
+
+#' @import data.table
+#' @import xgboost
+getTreeBreakdown = function(tree, col_names){
+  
+  ####accepts a tree (data table), and column names
+  ####outputs a data table, of the impact of each variable + intercept, for each leaf
+  
+ 
+  
+  tree_breakdown <- vector("list", length(col_names)  + 2)
+  names(tree_breakdown) = c(col_names,'intercept','leaf')
+
+  leaves = tree[leaf==T, Node]
+
+  for (leaf in leaves){
+
+    leaf_breakdown = getLeafBreakdown(tree,leaf,col_names)
+    leaf_breakdown$leaf = leaf
+    tree_breakdown = rbindlist(append(list(tree_breakdown),list(leaf_breakdown)))
+  }
+
+  return (tree_breakdown)
+}