simulations/measerr_methods.R

   1 library(formula.tools)
   2 library(matrixStats)
   3
   4 ## df: dataframe to model
   5 ## outcome_formula: formula for y | x, z
   6 ## outcome_family: family for y | x, z
   7 ## proxy_formula: formula for w | x, z, y
   8 ## proxy_family: family for w | x, z, y
   9 ## truth_formula: formula for x | z
  10 ## truth_family: family for x | z
  11
  12 ### ideal formulas for example 1
  13 # test.fit.1 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x, binomial(link='logit'), x ~ z)
  14
  15 ### ideal formulas for example 2
  16 # test.fit.2 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x + z + y + y:x, binomial(link='logit'), x ~ z)
  17
  18
  19 ## outcome_formula <- y ~ x + z; proxy_formula <- w_pred ~ y + x + z + x:z + x:y + z:y
  20 measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit')){
  21
  22     nll <- function(params){
  23         df.obs <- model.frame(outcome_formula, df)
  24         proxy.variable <- all.vars(proxy_formula)[1]
  25         proxy.model.matrix <- model.matrix(proxy_formula, df)
  26         response.var <- all.vars(outcome_formula)[1]
  27         y.obs <- with(df.obs,eval(parse(text=response.var)))
  28         outcome.model.matrix <- model.matrix(outcome_formula, df.obs)
  29
  30         param.idx <- 1
  31         n.outcome.model.covars <- dim(outcome.model.matrix)[2]
  32         outcome.params <- params[param.idx:n.outcome.model.covars]
  33         param.idx <- param.idx + n.outcome.model.covars
  34
  35         if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
  36             ll.y.obs <- vector(mode='numeric', length=length(y.obs))
  37             ll.y.obs[y.obs==1] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==1,]),log=TRUE)
  38             ll.y.obs[y.obs==0] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==0,]),log=TRUE,lower.tail=FALSE)
  39         }
  40
  41         df.obs <- model.frame(proxy_formula,df)
  42         n.proxy.model.covars <- dim(proxy.model.matrix)[2]
  43         proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
  44
  45         param.idx <- param.idx + n.proxy.model.covars
  46         proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
  47
  48         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
  49             ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
  50             ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
  51             ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
  52         }
  53
  54         ll.obs <- sum(ll.y.obs + ll.w.obs)
  55
  56         df.unobs <- df[is.na(df[[response.var]])]
  57         df.unobs.y1 <- copy(df.unobs)
  58         df.unobs.y1[[response.var]] <- 1
  59         df.unobs.y0 <- copy(df.unobs)
  60         df.unobs.y0[[response.var]] <- 0
  61
  62         ## integrate out y
  63         outcome.model.matrix.y1 <- model.matrix(outcome_formula, df.unobs.y1)
  64
  65         if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
  66             ll.y.unobs.1 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
  67             ll.y.unobs.0 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
  68             ll.y.unobs.1 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE)
  69             ll.y.unobs.0 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE,lower.tail=FALSE)
  70         }
  71
  72         proxy.model.matrix.y1 <- model.matrix(proxy_formula, df.unobs.y1)
  73         proxy.model.matrix.y0 <- model.matrix(proxy_formula, df.unobs.y0)
  74         proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
  75
  76         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
  77             ll.w.unobs.1 <- vector(mode='numeric',length=dim(proxy.model.matrix.y1)[1])
  78             ll.w.unobs.0 <- vector(mode='numeric',length=dim(proxy.model.matrix.y0)[1])
  79             ll.w.unobs.1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==1,]),log=TRUE)
  80             ll.w.unobs.1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
  81
  82             ll.w.unobs.0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==1,]),log=TRUE)
  83             ll.w.unobs.0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
  84         }
  85
  86         ll.unobs.1 <- ll.y.unobs.1 + ll.w.unobs.1
  87         ll.unobs.0 <- ll.y.unobs.0 + ll.w.unobs.0
  88         ll.unobs <- sum(colLogSumExps(rbind(ll.unobs.1,ll.unobs.0)))
  89         ll <- ll.unobs + ll.obs
  90         return(-ll)
  91     }
  92
  93     params <- colnames(model.matrix(outcome_formula,df))
  94     lower <- rep(-Inf, length(params))
  95     proxy.params <- colnames(model.matrix(proxy_formula, df))
  96     params <- c(params, paste0('proxy_',proxy.params))
  97     lower <- c(lower, rep(-Inf, length(proxy.params)))
  98     start <- rep(0.1,length(params))
  99     names(start) <- params
 100
 101     fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
 102     return(fit)
 103 }
 104
 105 measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit')){
 106
 107     measrr_mle_nll <- function(params){
 108         df.obs <- model.frame(outcome_formula, df)
 109
 110         proxy.variable <- all.vars(proxy_formula)[1]
 111         proxy.model.matrix <- model.matrix(proxy_formula, df)
 112
 113         response.var <- all.vars(outcome_formula)[1]
 114         y.obs <- with(df.obs,eval(parse(text=response.var)))
 115
 116         outcome.model.matrix <- model.matrix(outcome_formula, df)
 117
 118         param.idx <- 1
 119         n.outcome.model.covars <- dim(outcome.model.matrix)[2]
 120         outcome.params <- params[param.idx:n.outcome.model.covars]
 121         param.idx <- param.idx + n.outcome.model.covars
 122
 123         ## likelihood for the fully observed data
 124         if(outcome_family$family == "gaussian"){
 125             sigma.y <- params[param.idx]
 126             param.idx <- param.idx + 1
 127
 128             #  outcome_formula likelihood using linear regression
 129             ll.y.obs <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix),sd=sigma.y, log=TRUE)
 130         }
 131
 132         df.obs <- model.frame(proxy_formula,df)
 133         n.proxy.model.covars <- dim(proxy.model.matrix)[2]
 134         proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
 135         param.idx <- param.idx + n.proxy.model.covars
 136         proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
 137
 138         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
 139             ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
 140
 141             # proxy_formula likelihood using logistic regression
 142             ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
 143             ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
 144         }
 145
 146         df.obs <- model.frame(truth_formula, df)
 147         truth.variable <- all.vars(truth_formula)[1]
 148         truth.obs <- with(df.obs, eval(parse(text=truth.variable)))
 149         truth.model.matrix <- model.matrix(truth_formula,df)
 150         n.truth.model.covars <- dim(truth.model.matrix)[2]
 151
 152         truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]
 153
 154         if( (truth_family$family=="binomial") & (truth_family$link=='logit')){
 155             ll.x.obs <- vector(mode='numeric',length=dim(truth.model.matrix)[1])
 156
 157             # truth_formula likelihood using logistic regression
 158             ll.x.obs[truth.obs==1] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==1,]),log=TRUE)
 159             ll.x.obs[truth.obs==0] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==0,]),log=TRUE, lower.tail=FALSE)
 160         }
 161
 162         # add the three likelihoods
 163         ll.obs <- sum(ll.y.obs + ll.w.obs + ll.x.obs)
 164
 165         ## likelihood for the predicted data
 166         ## integrate out the "truth" variable.
 167
 168         if(truth_family$family=='binomial'){
 169             df.unobs <- df[is.na(eval(parse(text=truth.variable)))]
 170             df.unobs.x1 <- copy(df.unobs)
 171             df.unobs.x1[,'x'] <- 1
 172             df.unobs.x0 <- copy(df.unobs)
 173             df.unobs.x0[,'x'] <- 0
 174             outcome.unobs <- with(df.unobs, eval(parse(text=response.var)))
 175
 176             outcome.model.matrix.x0 <- model.matrix(outcome_formula, df.unobs.x0)
 177             outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1)
 178             if(outcome_family$family=="gaussian"){
 179
 180                 # likelihood of outcome
 181             ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
 182             ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
 183             }
 184
 185             if( (proxy_family$family=='binomial') & (proxy_family$link=='logit')){
 186
 187                 proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.unobs.x0)
 188                 proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.unobs.x1)
 189                 proxy.unobs <- df.unobs[[proxy.variable]]
 190                 ll.w.x0 <- vector(mode='numeric', length=dim(df.unobs)[1])
 191                 ll.w.x1 <- vector(mode='numeric', length=dim(df.unobs)[1])
 192
 193                 # likelihood of proxy
 194                 ll.w.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==1,]), log=TRUE)
 195                 ll.w.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==1,]), log=TRUE)
 196
 197                 ll.w.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
 198                 ll.w.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
 199             }
 200
 201             if(truth_family$link=='logit'){
 202                 truth.model.matrix <- model.matrix(truth_formula, df.unobs.x0)
 203                 # likelihood of truth
 204                 ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE)
 205                 ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE, lower.tail=FALSE)
 206             }
 207         }
 208
 209         ll.x0 <- ll.y.x0 + ll.w.x0 + ll.x.x0
 210         ll.x1 <- ll.y.x1 + ll.w.x1 + ll.x.x1
 211         ll.unobs <- sum(colLogSumExps(rbind(ll.x0, ll.x1)))
 212         return(-(ll.unobs + ll.obs))
 213     }
 214
 215     outcome.params <- colnames(model.matrix(outcome_formula,df))
 216     lower <- rep(-Inf, length(outcome.params))
 217
 218     if(outcome_family$family=='gaussian'){
 219         params <- c(outcome.params, 'sigma_y')
 220         lower <- c(lower, 0.00001)
 221     } else {
 222         params <- outcome.params
 223     }
 224
 225     proxy.params <- colnames(model.matrix(proxy_formula, df))
 226     params <- c(params, paste0('proxy_',proxy.params))
 227     lower <- c(lower, rep(-Inf, length(proxy.params)))
 228
 229     truth.params <- colnames(model.matrix(truth_formula, df))
 230     params <- c(params, paste0('truth_', truth.params))
 231     lower <- c(lower, rep(-Inf, length(truth.params)))
 232     start <- rep(0.1,length(params))
 233     names(start) <- params
 234
 235     fit <- optim(start, fn = measrr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
 236
 237     return(fit)
 238 }