library(formula.tools)
library(matrixStats)
+library(optimx)
library(bbmle)
## df: dataframe to model
## outcome_formula: formula for y | x, z
### ideal formulas for example 2
# test.fit.2 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x + z + y + y:x, binomial(link='logit'), x ~ z)
-
+likelihood.logistic <- function(model.params, outcome, model.matrix){
+ ll <- vector(mode='numeric', length=length(outcome))
+ ll[outcome == 1] <- plogis(model.params %*% t(model.matrix[outcome==1,]), log=TRUE)
+ ll[outcome == 0] <- plogis(model.params %*% t(model.matrix[outcome==0,]), log=TRUE, lower.tail=FALSE)
+ return(ll)
+}
## outcome_formula <- y ~ x + z; proxy_formula <- w_pred ~ y + x + z + x:z + x:y + z:y
measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit'),method='optim'){
+ df.obs <- model.frame(outcome_formula, df)
+ proxy.model.matrix <- model.matrix(proxy_formula, df)
+ proxy.variable <- all.vars(proxy_formula)[1]
+
+ df.proxy.obs <- model.frame(proxy_formula,df)
+ proxy.obs <- with(df.proxy.obs, eval(parse(text=proxy.variable)))
+
+ response.var <- all.vars(outcome_formula)[1]
+ y.obs <- with(df.obs,eval(parse(text=response.var)))
+ outcome.model.matrix <- model.matrix(outcome_formula, df.obs)
+
+ df.unobs <- df[is.na(df[[response.var]])]
+ df.unobs.y1 <- copy(df.unobs)
+ df.unobs.y1[[response.var]] <- 1
+ df.unobs.y0 <- copy(df.unobs)
+ df.unobs.y0[[response.var]] <- 0
+
+ outcome.model.matrix.y1 <- model.matrix(outcome_formula, df.unobs.y1)
+ proxy.model.matrix.y1 <- model.matrix(proxy_formula, df.unobs.y1)
+ proxy.model.matrix.y0 <- model.matrix(proxy_formula, df.unobs.y0)
+ proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
nll <- function(params){
- df.obs <- model.frame(outcome_formula, df)
- proxy.variable <- all.vars(proxy_formula)[1]
- proxy.model.matrix <- model.matrix(proxy_formula, df)
- response.var <- all.vars(outcome_formula)[1]
- y.obs <- with(df.obs,eval(parse(text=response.var)))
- outcome.model.matrix <- model.matrix(outcome_formula, df.obs)
param.idx <- 1
n.outcome.model.covars <- dim(outcome.model.matrix)[2]
ll.y.obs[y.obs==0] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==0,]),log=TRUE,lower.tail=FALSE)
}
- df.obs <- model.frame(proxy_formula,df)
n.proxy.model.covars <- dim(proxy.model.matrix)[2]
proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
-
param.idx <- param.idx + n.proxy.model.covars
- proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
}
ll.obs <- sum(ll.y.obs + ll.w.obs)
-
- df.unobs <- df[is.na(df[[response.var]])]
- df.unobs.y1 <- copy(df.unobs)
- df.unobs.y1[[response.var]] <- 1
- df.unobs.y0 <- copy(df.unobs)
- df.unobs.y0[[response.var]] <- 0
## integrate out y
- outcome.model.matrix.y1 <- model.matrix(outcome_formula, df.unobs.y1)
if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
ll.y.unobs.1 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
ll.y.unobs.0 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE,lower.tail=FALSE)
}
- proxy.model.matrix.y1 <- model.matrix(proxy_formula, df.unobs.y1)
- proxy.model.matrix.y0 <- model.matrix(proxy_formula, df.unobs.y0)
- proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
-
if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
ll.w.unobs.1 <- vector(mode='numeric',length=dim(proxy.model.matrix.y1)[1])
ll.w.unobs.0 <- vector(mode='numeric',length=dim(proxy.model.matrix.y0)[1])
return(fit)
}
-## Experimental, and not necessary if errors are independent.
-measerr_irr_mle <- function(df, outcome_formula, outcome_family=gaussian(), rater_formula, proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){
- ### in this scenario, the ground truth also has measurement error, but we have repeated measures for it.
+measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){
- ## probability of y given observed data.
- df.obs <- df[!is.na(x.obs.1)]
+ df.obs <- model.frame(outcome_formula, df)
+ response.var <- all.vars(outcome_formula)[1]
proxy.variable <- all.vars(proxy_formula)[1]
- df.x.obs.1 <- copy(df.obs)[,x:=1]
- df.x.obs.0 <- copy(df.obs)[,x:=0]
- y.obs <- df.obs[,y]
+ truth.variable <- all.vars(truth_formula)[1]
+ outcome.model.matrix <- model.matrix(outcome_formula, df)
+ proxy.model.matrix <- model.matrix(proxy_formula, df)
+ y.obs <- with(df.obs,eval(parse(text=response.var)))
+
+ df.proxy.obs <- model.frame(proxy_formula,df)
+ proxy.obs <- with(df.proxy.obs, eval(parse(text=proxy.variable)))
+ n.proxy.model.covars <- dim(proxy.model.matrix)[2]
+
+ df.truth.obs <- model.frame(truth_formula, df)
+ truth.obs <- with(df.truth.obs, eval(parse(text=truth.variable)))
+ truth.model.matrix <- model.matrix(truth_formula,df.truth.obs)
+ n.truth.model.covars <- dim(truth.model.matrix)[2]
+
+ df.unobs <- df[is.na(eval(parse(text=truth.variable)))]
+ df.unobs.x1 <- copy(df.unobs)
+ df.unobs.x1[,truth.variable] <- 1
+ df.unobs.x0 <- copy(df.unobs)
+ df.unobs.x0[,truth.variable] <- 0
+ outcome.unobs <- with(df.unobs, eval(parse(text=response.var)))
+
+ outcome.model.matrix.x0 <- model.matrix(outcome_formula, df.unobs.x0)
+ outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1)
+
+ proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.unobs.x0)
+ proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.unobs.x1)
+ proxy.unobs <- df.unobs[[proxy.variable]]
- nll <- function(params){
- outcome.model.matrix.x.obs.0 <- model.matrix(outcome_formula, df.x.obs.0)
- outcome.model.matrix.x.obs.1 <- model.matrix(outcome_formula, df.x.obs.1)
+ truth.model.matrix.unobs <- model.matrix(truth_formula, df.unobs.x0)
+ measerr_mle_nll <- function(params){
param.idx <- 1
- n.outcome.model.covars <- dim(outcome.model.matrix.x.obs.0)[2]
+ n.outcome.model.covars <- dim(outcome.model.matrix)[2]
outcome.params <- params[param.idx:n.outcome.model.covars]
param.idx <- param.idx + n.outcome.model.covars
- sigma.y <- params[param.idx]
- param.idx <- param.idx + 1
-
- ll.y.x.obs.0 <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix.x.obs.0),sd=sigma.y, log=TRUE)
- ll.y.x.obs.1 <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix.x.obs.1),sd=sigma.y, log=TRUE)
-
- ## assume that the two coders are statistically independent conditional on x
- ll.x.obs.0.x0 <- vector(mode='numeric', length=nrow(df.obs))
- ll.x.obs.1.x0 <- vector(mode='numeric', length=nrow(df.obs))
- ll.x.obs.0.x1 <- vector(mode='numeric', length=nrow(df.obs))
- ll.x.obs.1.x1 <- vector(mode='numeric', length=nrow(df.obs))
-
- rater.model.matrix.x.obs.0 <- model.matrix(rater_formula, df.x.obs.0)
- rater.model.matrix.x.obs.1 <- model.matrix(rater_formula, df.x.obs.1)
-
- n.rater.model.covars <- dim(rater.model.matrix.x.obs.0)[2]
- rater.0.params <- params[param.idx:(n.rater.model.covars + param.idx - 1)]
- param.idx <- param.idx + n.rater.model.covars
+ ## likelihood for the fully observed data
+ if(outcome_family$family == "gaussian"){
+ sigma.y <- params[param.idx]
+ param.idx <- param.idx + 1
+ # outcome_formula likelihood using linear regression
+ ll.y.obs <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix),sd=sigma.y, log=TRUE)
+ } else if( (outcome_family$family == "binomial") & (outcome_family$link == "logit") )
+ ll.y.obs <- likelihood.logistic(outcome.params, y.obs, outcome.model.matrix)
- rater.1.params <- params[param.idx:(n.rater.model.covars + param.idx - 1)]
- param.idx <- param.idx + n.rater.model.covars
-
- # probability of rater 0 if x is 0 or 1
- ll.x.obs.0.x0[df.obs$x.obs.0==1] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.0==1,]), log=TRUE)
- ll.x.obs.0.x0[df.obs$x.obs.0==0] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.0==0,]), log=TRUE, lower.tail=FALSE)
- ll.x.obs.0.x1[df.obs$x.obs.0==1] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.0==1,]), log=TRUE)
- ll.x.obs.0.x1[df.obs$x.obs.0==0] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.0==0,]), log=TRUE, lower.tail=FALSE)
-
- # probability of rater 1 if x is 0 or 1
- ll.x.obs.1.x0[df.obs$x.obs.1==1] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.1==1,]), log=TRUE)
- ll.x.obs.1.x0[df.obs$x.obs.1==0] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.1==0,]), log=TRUE, lower.tail=FALSE)
- ll.x.obs.1.x1[df.obs$x.obs.1==1] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.1==1,]), log=TRUE)
- ll.x.obs.1.x1[df.obs$x.obs.1==0] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.1==0,]), log=TRUE, lower.tail=FALSE)
-
- proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.x.obs.0)
- proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.x.obs.1)
-
- n.proxy.model.covars <- dim(proxy.model.matrix.x0)[2]
+
proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
param.idx <- param.idx + n.proxy.model.covars
- proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
-
- if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
- ll.w.obs.x0 <- vector(mode='numeric',length=dim(proxy.model.matrix.x0)[1])
- ll.w.obs.x1 <- vector(mode='numeric',length=dim(proxy.model.matrix.x1)[1])
+ if( (proxy_family$family=="binomial") & (proxy_family$link=='logit'))
+ ll.w.obs <- likelihood.logistic(proxy.params, proxy.obs, proxy.model.matrix)
- # proxy_formula likelihood using logistic regression
- ll.w.obs.x0[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.obs==1,]),log=TRUE)
- ll.w.obs.x0[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
-
- ll.w.obs.x1[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.obs==1,]),log=TRUE)
- ll.w.obs.x1[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
- }
-
- ## assume that the probability of x is a logistic regression depending on z
- truth.model.matrix.obs <- model.matrix(truth_formula, df.obs)
- n.truth.params <- dim(truth.model.matrix.obs)[2]
- truth.params <- params[param.idx:(n.truth.params + param.idx - 1)]
-
- ll.obs.x0 <- plogis(truth.params %*% t(truth.model.matrix.obs), log=TRUE)
- ll.obs.x1 <- plogis(truth.params %*% t(truth.model.matrix.obs), log=TRUE, lower.tail=FALSE)
-
- ll.obs <- colLogSumExps(rbind(ll.y.x.obs.0 + ll.x.obs.0.x0 + ll.x.obs.1.x0 + ll.obs.x0 + ll.w.obs.x0,
- ll.y.x.obs.1 + ll.x.obs.0.x1 + ll.x.obs.1.x1 + ll.obs.x1 + ll.w.obs.x1))
+ truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]
- ### NOW FOR THE FUN PART. Likelihood of the unobserved data.
- ### we have to integrate out x.obs.0, x.obs.1, and x.
+ if( (truth_family$family=="binomial") & (truth_family$link=='logit'))
+ ll.x.obs <- likelihood.logistic(truth.params, truth.obs, truth.model.matrix)
+
+ # add the three likelihoods
+ ll.obs <- sum(ll.y.obs + ll.w.obs + ll.x.obs)
+ ## likelihood for the predicted data
+ ## integrate out the "truth" variable.
+
+ if(truth_family$family=='binomial'){
+ if(outcome_family$family=="gaussian"){
+ # likelihood of outcome
+ ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
+ ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
+ } else if( (outcome_family$family == "binomial") & (outcome_family$link == "logit") ){
+ ll.y.x1 <- likelihood.logistic(outcome.params, outcome.unobs, outcome.model.matrix.x1)
+ ll.y.x0 <- likelihood.logistic(outcome.params, outcome.unobs, outcome.model.matrix.x0)
+ }
- ## THE OUTCOME
- df.unobs <- df[is.na(x.obs)]
- df.x.unobs.0 <- copy(df.unobs)[,x:=0]
- df.x.unobs.1 <- copy(df.unobs)[,x:=1]
- y.unobs <- df.unobs$y
+ if( (proxy_family$family=='binomial') & (proxy_family$link=='logit')){
- outcome.model.matrix.x.unobs.0 <- model.matrix(outcome_formula, df.x.unobs.0)
- outcome.model.matrix.x.unobs.1 <- model.matrix(outcome_formula, df.x.unobs.1)
+ ll.w.x0 <- likelihood.logistic(proxy.params, proxy.unobs, proxy.model.matrix.x0)
+ ll.w.x1 <- likelihood.logistic(proxy.params, proxy.unobs, proxy.model.matrix.x1)
- ll.y.unobs.x0 <- dnorm(y.unobs, outcome.params %*% t(outcome.model.matrix.x.unobs.0), sd=sigma.y, log=TRUE)
- ll.y.unobs.x1 <- dnorm(y.unobs, outcome.params %*% t(outcome.model.matrix.x.unobs.1), sd=sigma.y, log=TRUE)
+ }
-
- ## THE UNLABELED DATA
+ if(truth_family$link=='logit'){
+ # likelihood of truth
+ ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE)
+ ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE, lower.tail=FALSE)
+ }
+ }
-
- ## assume that the two coders are statistically independent conditional on x
- ll.x.unobs.0.x0 <- vector(mode='numeric', length=nrow(df.unobs))
- ll.x.unobs.1.x0 <- vector(mode='numeric', length=nrow(df.unobs))
- ll.x.unobs.0.x1 <- vector(mode='numeric', length=nrow(df.unobs))
- ll.x.unobs.1.x1 <- vector(mode='numeric', length=nrow(df.unobs))
-
- df.x.unobs.0[,x.obs := 1]
- df.x.unobs.1[,x.obs := 1]
+ ll.x0 <- ll.y.x0 + ll.w.x0 + ll.x.x0
+ ll.x1 <- ll.y.x1 + ll.w.x1 + ll.x.x1
+ ll.unobs <- sum(colLogSumExps(rbind(ll.x0, ll.x1)))
+ return(-(ll.unobs + ll.obs))
+ }
+
+ outcome.params <- colnames(model.matrix(outcome_formula,df))
+ lower <- rep(-Inf, length(outcome.params))
- rater.model.matrix.x.unobs.0 <- model.matrix(rater_formula, df.x.unobs.0)
- rater.model.matrix.x.unobs.1 <- model.matrix(rater_formula, df.x.unobs.1)
+ if(outcome_family$family=='gaussian'){
+ params <- c(outcome.params, 'sigma_y')
+ lower <- c(lower, 0.00001)
+ } else {
+ params <- outcome.params
+ }
+
+ proxy.params <- colnames(model.matrix(proxy_formula, df))
+ params <- c(params, paste0('proxy_',proxy.params))
+ lower <- c(lower, rep(-Inf, length(proxy.params)))
-
- ## # probability of rater 0 if x is 0 or 1
- ## ll.x.unobs.0.x0 <- colLogSumExps(rbind(plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE),
- ## plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE, lower.tail=TRUE)))
+ truth.params <- colnames(model.matrix(truth_formula, df))
+ params <- c(params, paste0('truth_', truth.params))
+ lower <- c(lower, rep(-Inf, length(truth.params)))
+ start <- rep(0.1,length(params))
+ names(start) <- params
- ## ll.x.unobs.0.x1 <- colLogSumExps(rbind(plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE),
- ## plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE, lower.tail=TRUE)))
+ if(method=='optim'){
+ fit <- optim(start, fn = measerr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+ } else { # method='mle2'
+
+ quoted.names <- gsub("[\\(\\)]",'',names(start))
- ## # probability of rater 1 if x is 0 or 1
- ## ll.x.unobs.1.x0 <- colLogSumExps(rbind(plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE),
- ## plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE, lower.tail=TRUE)))
+ text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(measerr_mle_nll(params))}")
- ## ll.x.unobs.1.x1 <- colLogSumExps(rbind(plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE),
- ## plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE, lower.tail=TRUE)))
+ measerr_mle_nll_mle <- eval(parse(text=text))
+ names(start) <- quoted.names
+ names(lower) <- quoted.names
+ fit <- mle2(minuslogl=measerr_mle_nll_mle, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
+ }
+ return(fit)
+}
- proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
- proxy.model.matrix.x0.unobs <- model.matrix(proxy_formula, df.x.unobs.0)
- proxy.model.matrix.x1.unobs <- model.matrix(proxy_formula, df.x.unobs.1)
+## Experimental, but probably works.
+measerr_irr_mle <- function(df, outcome_formula, outcome_family=gaussian(), coder_formulas=list(x.obs.0~x, x.obs.1~x), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){
- if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
- ll.w.unobs.x0 <- vector(mode='numeric',length=dim(proxy.model.matrix.x0)[1])
- ll.w.unobs.x1 <- vector(mode='numeric',length=dim(proxy.model.matrix.x1)[1])
+ ### in this scenario, the ground truth also has measurement error, but we have repeated measures for it.
+ # this time we never get to observe the true X
+ outcome.model.matrix <- model.matrix(outcome_formula, df)
+ proxy.model.matrix <- model.matrix(proxy_formula, df)
+ response.var <- all.vars(outcome_formula)[1]
+ proxy.var <- all.vars(proxy_formula)[1]
+ param.var <- all.vars(truth_formula)[1]
+ truth.var<- all.vars(truth_formula)[1]
+ y <- with(df,eval(parse(text=response.var)))
+ nll <- function(params){
+ param.idx <- 1
+ n.outcome.model.covars <- dim(outcome.model.matrix)[2]
+ outcome.params <- params[param.idx:n.outcome.model.covars]
+ param.idx <- param.idx + n.outcome.model.covars
- # proxy_formula likelihood using logistic regression
- ll.w.unobs.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0.unobs[proxy.unobs==1,]),log=TRUE)
- ll.w.unobs.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0.unobs[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
- ll.w.unobs.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1.unobs[proxy.unobs==1,]),log=TRUE)
- ll.w.unobs.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1.unobs[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
+ if(outcome_family$family == "gaussian"){
+ sigma.y <- params[param.idx]
+ param.idx <- param.idx + 1
}
- truth.model.matrix.unobs <- model.matrix(truth_formula, df.unobs)
- ll.unobs.x0 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE)
- ll.unobs.x1 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE, lower.tail=FALSE)
+ n.proxy.model.covars <- dim(proxy.model.matrix)[2]
+ proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
+ param.idx <- param.idx + n.proxy.model.covars
+
+ df.temp <- copy(df)
+
+ if((truth_family$family == "binomial")
+ & (truth_family$link=='logit')){
+ integrate.grid <- expand.grid(replicate(1 + length(coder_formulas), c(0,1), simplify=FALSE))
+ ll.parts <- matrix(nrow=nrow(df),ncol=nrow(integrate.grid))
+ for(i in 1:nrow(integrate.grid)){
+ # setup the dataframe for this row
+ row <- integrate.grid[i,]
+
+ df.temp[[param.var]] <- row[[1]]
+ ci <- 2
+ for(coder_formula in coder_formulas){
+ coder.var <- all.vars(coder_formula)[1]
+ df.temp[[coder.var]] <- row[[ci]]
+ ci <- ci + 1
+ }
+
+ outcome.model.matrix <- model.matrix(outcome_formula, df.temp)
+ if(outcome_family$family == "gaussian"){
+ ll.y <- dnorm(y, outcome.params %*% t(outcome.model.matrix), sd=sigma.y, log=TRUE)
+ }
+
+ if(proxy_family$family=="binomial" & (proxy_family$link=='logit')){
+ proxy.model.matrix <- model.matrix(proxy_formula, df.temp)
+ ll.w <- vector(mode='numeric', length=dim(proxy.model.matrix)[1])
+ proxyvar <- with(df.temp,eval(parse(text=proxy.var)))
+ ll.w[proxyvar==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxyvar==1,]),log=TRUE)
+ ll.w[proxyvar==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxyvar==0,]),log=TRUE,lower.tail=FALSE)
+ }
+
+ ## probability of the coded variables
+ coder.lls <- matrix(nrow=nrow(df.temp),ncol=length(coder_formulas))
+ ci <- 1
+ for(coder_formula in coder_formulas){
+ coder.model.matrix <- model.matrix(coder_formula, df.temp)
+ n.coder.model.covars <- dim(coder.model.matrix)[2]
+ coder.params <- params[param.idx:(n.coder.model.covars + param.idx - 1)]
+ param.idx <- param.idx + n.coder.model.covars
+ coder.var <- all.vars(coder_formula)[1]
+ x.obs <- with(df.temp, eval(parse(text=coder.var)))
+ true.codervar <- df[[all.vars(coder_formula)[1]]]
+
+ ll.coder <- vector(mode='numeric', length=dim(coder.model.matrix)[1])
+ ll.coder[x.obs==1] <- plogis(coder.params %*% t(coder.model.matrix[x.obs==1,]),log=TRUE)
+ ll.coder[x.obs==0] <- plogis(coder.params %*% t(coder.model.matrix[x.obs==0,]),log=TRUE,lower.tail=FALSE)
+
+ # don't count when we know the observed value, unless we're accounting for observed value
+ ll.coder[(!is.na(true.codervar)) & (true.codervar != x.obs)] <- NA
+ coder.lls[,ci] <- ll.coder
+ ci <- ci + 1
+ }
+
+ truth.model.matrix <- model.matrix(truth_formula, df.temp)
+ n.truth.model.covars <- dim(truth.model.matrix)[2]
+ truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]
+
+ for(coder_formula in coder_formulas){
+ coder.model.matrix <- model.matrix(coder_formula, df.temp)
+ n.coder.model.covars <- dim(coder.model.matrix)[2]
+ param.idx <- param.idx - n.coder.model.covars
+ }
+
+ x <- with(df.temp, eval(parse(text=truth.var)))
+ ll.truth <- vector(mode='numeric', length=dim(truth.model.matrix)[1])
+ ll.truth[x==1] <- plogis(truth.params %*% t(truth.model.matrix[x==1,]), log=TRUE)
+ ll.truth[x==0] <- plogis(truth.params %*% t(truth.model.matrix[x==0,]), log=TRUE, lower.tail=FALSE)
+
+ true.truthvar <- df[[all.vars(truth_formula)[1]]]
+
+ if(!is.null(true.truthvar)){
+ # ll.truth[(!is.na(true.truthvar)) & (true.truthvar != truthvar)] <- -Inf
+ # ll.truth[(!is.na(true.truthvar)) & (true.truthvar == truthvar)] <- 0
+ }
+ ll.parts[,i] <- ll.y + ll.w + apply(coder.lls,1,sum) + ll.truth
+
+ }
- ll.unobs <- colLogSumExps(rbind(ll.unobs.x0 + ll.w.unobs.x0 + ll.y.unobs.x0,
- ll.unobs.x1 + ll.w.unobs.x1 + ll.y.unobs.x1))
+ lls <- rowLogSumExps(ll.parts,na.rm=TRUE)
- return(-1 *( sum(ll.obs) + sum(ll.unobs)))
+ ## likelihood of observed data
+ target <- -1 * sum(lls)
+ return(target)
+ }
}
-
+
outcome.params <- colnames(model.matrix(outcome_formula,df))
lower <- rep(-Inf, length(outcome.params))
} else {
params <- outcome.params
}
-
- rater.0.params <- colnames(model.matrix(rater_formula,df))
- params <- c(params, paste0('rater_0',rater.0.params))
- lower <- c(lower, rep(-Inf, length(rater.0.params)))
-
- rater.1.params <- colnames(model.matrix(rater_formula,df))
- params <- c(params, paste0('rater_1',rater.1.params))
- lower <- c(lower, rep(-Inf, length(rater.1.params)))
proxy.params <- colnames(model.matrix(proxy_formula, df))
params <- c(params, paste0('proxy_',proxy.params))
+ positive.params <- paste0('proxy_',truth.var)
lower <- c(lower, rep(-Inf, length(proxy.params)))
+ names(lower) <- params
+ lower[positive.params] <- 0.01
+ ci <- 0
+
+ for(coder_formula in coder_formulas){
+ coder.params <- colnames(model.matrix(coder_formula,df))
+ params <- c(params, paste0('coder_',ci,coder.params))
+ positive.params <- paste0('coder_', ci, truth.var)
+ ci <- ci + 1
+ lower <- c(lower, rep(-Inf, length(coder.params)))
+ names(lower) <- params
+ lower[positive.params] <- 0.01
+ }
truth.params <- colnames(model.matrix(truth_formula, df))
params <- c(params, paste0('truth_', truth.params))
lower <- c(lower, rep(-Inf, length(truth.params)))
start <- rep(0.1,length(params))
names(start) <- params
-
+ names(lower) <- params
if(method=='optim'){
+ print(start)
fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
} else {
measerr_mle_nll <- eval(parse(text=text))
names(start) <- quoted.names
names(lower) <- quoted.names
- fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
+ fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, method='L-BFGS-B',control=list(maxit=1e6))
}
return(fit)
}
+## Experimental, and does not work.
+measerr_irr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), coder_formulas=list(y.obs.0~y+w_pred+y.obs.1,y.obs.1~y+w_pred+y.obs.0), proxy_formula=w_pred~y, proxy_family=binomial(link='logit'),method='optim'){
+ integrate.grid <- expand.grid(replicate(1 + length(coder_formulas), c(0,1), simplify=FALSE))
+# print(integrate.grid)
-measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){
- measerr_mle_nll <- function(params){
- df.obs <- model.frame(outcome_formula, df)
- proxy.variable <- all.vars(proxy_formula)[1]
- proxy.model.matrix <- model.matrix(proxy_formula, df)
- response.var <- all.vars(outcome_formula)[1]
- y.obs <- with(df.obs,eval(parse(text=response.var)))
+ outcome.model.matrix <- model.matrix(outcome_formula, df)
+ n.outcome.model.covars <- dim(outcome.model.matrix)[2]
- outcome.model.matrix <- model.matrix(outcome_formula, df)
+ ### in this scenario, the ground truth also has measurement error, but we have repeated measures for it.
+ # this time we never get to observe the true X
+ nll <- function(params){
param.idx <- 1
- n.outcome.model.covars <- dim(outcome.model.matrix)[2]
outcome.params <- params[param.idx:n.outcome.model.covars]
param.idx <- param.idx + n.outcome.model.covars
+ proxy.model.matrix <- model.matrix(proxy_formula, df)
+ n.proxy.model.covars <- dim(proxy.model.matrix)[2]
+ response.var <- all.vars(outcome_formula)[1]
- ## likelihood for the fully observed data
if(outcome_family$family == "gaussian"){
sigma.y <- params[param.idx]
param.idx <- param.idx + 1
-
- # outcome_formula likelihood using linear regression
- ll.y.obs <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix),sd=sigma.y, log=TRUE)
}
-
- df.obs <- model.frame(proxy_formula,df)
- n.proxy.model.covars <- dim(proxy.model.matrix)[2]
+
proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
param.idx <- param.idx + n.proxy.model.covars
- proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
- if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
- ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
-
- # proxy_formula likelihood using logistic regression
- ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
- ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
- }
-
- df.obs <- model.frame(truth_formula, df)
- truth.variable <- all.vars(truth_formula)[1]
- truth.obs <- with(df.obs, eval(parse(text=truth.variable)))
- truth.model.matrix <- model.matrix(truth_formula,df)
- n.truth.model.covars <- dim(truth.model.matrix)[2]
-
- truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]
-
- if( (truth_family$family=="binomial") & (truth_family$link=='logit')){
- ll.x.obs <- vector(mode='numeric',length=dim(truth.model.matrix)[1])
-
- # truth_formula likelihood using logistic regression
- ll.x.obs[truth.obs==1] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==1,]),log=TRUE)
- ll.x.obs[truth.obs==0] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==0,]),log=TRUE, lower.tail=FALSE)
- }
-
- # add the three likelihoods
- ll.obs <- sum(ll.y.obs + ll.w.obs + ll.x.obs)
-
- ## likelihood for the predicted data
- ## integrate out the "truth" variable.
-
- if(truth_family$family=='binomial'){
- df.unobs <- df[is.na(eval(parse(text=truth.variable)))]
- df.unobs.x1 <- copy(df.unobs)
- df.unobs.x1[,'x'] <- 1
- df.unobs.x0 <- copy(df.unobs)
- df.unobs.x0[,'x'] <- 0
- outcome.unobs <- with(df.unobs, eval(parse(text=response.var)))
-
- outcome.model.matrix.x0 <- model.matrix(outcome_formula, df.unobs.x0)
- outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1)
- if(outcome_family$family=="gaussian"){
-
- # likelihood of outcome
- ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
- ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
+ df.temp <- copy(df)
+
+ if((outcome_family$family == "binomial")
+ & (outcome_family$link=='logit')){
+ ll.parts <- matrix(nrow=nrow(df),ncol=nrow(integrate.grid))
+ for(i in 1:nrow(integrate.grid)){
+ # setup the dataframe for this row
+ row <- integrate.grid[i,]
+
+ df.temp[[response.var]] <- row[[1]]
+ ci <- 2
+ for(coder_formula in coder_formulas){
+ codervar <- all.vars(coder_formula)[1]
+ df.temp[[codervar]] <- row[[ci]]
+ ci <- ci + 1
+ }
+
+ outcome.model.matrix <- model.matrix(outcome_formula, df.temp)
+ if(outcome_family$family == "gaussian"){
+ ll.y <- dnorm(df.temp[[response.var]], outcome.params %*% t(outcome.model.matrix), sd=sigma.y, log=T)
+ }
+
+ if(outcome_family$family == "binomial" & (outcome_family$link=='logit')){
+ ll.y <- vector(mode='numeric',length=nrow(df.temp))
+ ll.y[df.temp[[response.var]]==1] <- plogis( outcome.params %*% t(outcome.model.matrix), log=TRUE)
+ ll.y[df.temp[[response.var]]==0] <- plogis( outcome.params %*% t(outcome.model.matrix), log=TRUE,lower.tail=FALSE)
+ }
+
+ if(proxy_family$family=="binomial" & (proxy_family$link=='logit')){
+ proxy.model.matrix <- model.matrix(proxy_formula, df.temp)
+ ll.w <- vector(mode='numeric', length=dim(proxy.model.matrix)[1])
+ proxyvar <- with(df.temp,eval(parse(text=all.vars(proxy_formula)[1])))
+ ll.w[proxyvar==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxyvar==1,]),log=TRUE)
+ ll.w[proxyvar==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxyvar==0,]),log=TRUE,lower.tail=FALSE)
+ }
+
+ ## probability of the coded variables
+ coder.lls <- matrix(nrow=nrow(df.temp),ncol=length(coder_formulas))
+ ci <- 1
+ for(coder_formula in coder_formulas){
+ coder.model.matrix <- model.matrix(coder_formula, df.temp)
+ n.coder.model.covars <- dim(coder.model.matrix)[2]
+ coder.params <- params[param.idx:(n.coder.model.covars + param.idx - 1)]
+ param.idx <- param.idx + n.coder.model.covars
+ codervar <- with(df.temp, eval(parse(text=all.vars(coder_formula)[1])))
+ true.codervar <- df[[all.vars(coder_formula)[1]]]
+
+ ll.coder <- vector(mode='numeric', length=dim(coder.model.matrix)[1])
+ ll.coder[codervar==1] <- plogis(coder.params %*% t(coder.model.matrix[codervar==1,]),log=TRUE)
+ ll.coder[codervar==0] <- plogis(coder.params %*% t(coder.model.matrix[codervar==0,]),log=TRUE,lower.tail=FALSE)
+
+ # don't count when we know the observed value, unless we're accounting for observed value
+ ll.coder[(!is.na(true.codervar)) & (true.codervar != codervar)] <- NA
+ coder.lls[,ci] <- ll.coder
+ ci <- ci + 1
+ }
+
+ for(coder_formula in coder_formulas){
+ coder.model.matrix <- model.matrix(coder_formula, df.temp)
+ n.coder.model.covars <- dim(coder.model.matrix)[2]
+ param.idx <- param.idx - n.coder.model.covars
+ }
+
+ ll.parts[,i] <- ll.y + ll.w + apply(coder.lls,1,function(x) sum(x))
+
}
- if( (proxy_family$family=='binomial') & (proxy_family$link=='logit')){
-
- proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.unobs.x0)
- proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.unobs.x1)
- proxy.unobs <- df.unobs[[proxy.variable]]
- ll.w.x0 <- vector(mode='numeric', length=dim(df.unobs)[1])
- ll.w.x1 <- vector(mode='numeric', length=dim(df.unobs)[1])
-
- # likelihood of proxy
- ll.w.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==1,]), log=TRUE)
- ll.w.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==1,]), log=TRUE)
-
- ll.w.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
- ll.w.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
- }
+ lls <- rowLogSumExps(ll.parts,na.rm=TRUE)
- if(truth_family$link=='logit'){
- truth.model.matrix <- model.matrix(truth_formula, df.unobs.x0)
- # likelihood of truth
- ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE)
- ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE, lower.tail=FALSE)
- }
+ ## likelihood of observed data
+ target <- -1 * sum(lls)
+# print(target)
+# print(params)
+ return(target)
}
-
- ll.x0 <- ll.y.x0 + ll.w.x0 + ll.x.x0
- ll.x1 <- ll.y.x1 + ll.w.x1 + ll.x.x1
- ll.unobs <- sum(colLogSumExps(rbind(ll.x0, ll.x1)))
- return(-(ll.unobs + ll.obs))
}
-
+
outcome.params <- colnames(model.matrix(outcome_formula,df))
+ response.var <- all.vars(outcome_formula)[1]
lower <- rep(-Inf, length(outcome.params))
if(outcome_family$family=='gaussian'){
} else {
params <- outcome.params
}
-
+
+ ## constrain the model of the coder and proxy vars
+ ## this is to ensure identifiability
+ ## it is a safe assumption because the coders aren't hostile (wrong more often than right)
+ ## so we can assume that y ~Bw, B is positive
proxy.params <- colnames(model.matrix(proxy_formula, df))
+ positive.params <- paste0('proxy_',response.var)
params <- c(params, paste0('proxy_',proxy.params))
lower <- c(lower, rep(-Inf, length(proxy.params)))
+ names(lower) <- params
+ lower[positive.params] <- 0.001
+
+ ci <- 0
+ for(coder_formula in coder_formulas){
+ coder.params <- colnames(model.matrix(coder_formula,df))
+ latent.coder.params <- coder.params %in% response.var
+ params <- c(params, paste0('coder_',ci,coder.params))
+ positive.params <- paste0('coder_',ci,response.var)
+ ci <- ci + 1
+ lower <- c(lower, rep(-Inf, length(coder.params)))
+ names(lower) <-params
+ lower[positive.params] <- 0.001
+ }
- truth.params <- colnames(model.matrix(truth_formula, df))
- params <- c(params, paste0('truth_', truth.params))
- lower <- c(lower, rep(-Inf, length(truth.params)))
- start <- rep(0.1,length(params))
+ ## init by using the "loco model"
+ temp.df <- copy(df)
+ temp.df <- temp.df[y.obs.1 == y.obs.0, y:=y.obs.1]
+ loco.model <- glm(outcome_formula, temp.df, family=outcome_family)
+
+ start <- rep(1,length(params))
names(start) <- params
-
+ start[names(coef(loco.model))] <- coef(loco.model)
+ names(lower) <- params
if(method=='optim'){
- fit <- optim(start, fn = measerr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
- } else { # method='mle2'
+ print(lower)
+ fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE,control=list(maxit=1e6))
+ } else {
quoted.names <- gsub("[\\(\\)]",'',names(start))
+ print(quoted.names)
+ text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(nll(params))}")
- text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(measerr_mle_nll(params))}")
-
- measerr_mle_nll_mle <- eval(parse(text=text))
+ measerr_mle_nll <- eval(parse(text=text))
names(start) <- quoted.names
names(lower) <- quoted.names
- fit <- mle2(minuslogl=measerr_mle_nll_mle, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
+ fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
}
return(fit)
projects / ml_measurement_error_public.git / blobdiff