Make summarize estimator group correctly for robustness checks.

[ml_measurement_error_public.git] / simulations / measerr_methods.R

diff --git a/simulations/measerr_methods.R b/simulations/measerr_methods.R
index 00f1746e8e01228a26c11c7dbe3ea8b2673f81f7..92309edaab1bde2e4928cb0cc008fda6725a4b91 100644 (file)
--- a/simulations/measerr_methods.R
+++ b/simulations/measerr_methods.R
@@ -1,6 +1,7 @@
 library(formula.tools)
 library(matrixStats)
 library(formula.tools)
 library(matrixStats)

-
+library(optimx)
+library(bbmle)

 ## df: dataframe to model
 ## outcome_formula: formula for y | x, z
 ## outcome_family: family for y | x, z
 ## df: dataframe to model
 ## outcome_formula: formula for y | x, z
 ## outcome_family: family for y | x, z


@@ -14,18 +15,38 @@ library(matrixStats)
 
 ### ideal formulas for example 2
 # test.fit.2 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x + z + y + y:x, binomial(link='logit'), x ~ z)
 
 ### ideal formulas for example 2
 # test.fit.2 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x + z + y + y:x, binomial(link='logit'), x ~ z)

-
+likelihood.logistic <- function(model.params, outcome, model.matrix){
+    ll <- vector(mode='numeric', length=length(outcome))
+    ll[outcome == 1] <- plogis(model.params %*% t(model.matrix[outcome==1,]), log=TRUE)
+    ll[outcome == 0] <- plogis(model.params %*% t(model.matrix[outcome==0,]), log=TRUE, lower.tail=FALSE)
+    return(ll)
+}

 
 ## outcome_formula <- y ~ x + z; proxy_formula <- w_pred ~ y + x + z + x:z + x:y + z:y 
 
 ## outcome_formula <- y ~ x + z; proxy_formula <- w_pred ~ y + x + z + x:z + x:y + z:y 

-measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit')){
+measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit'),method='optim'){
+    df.obs <- model.frame(outcome_formula, df)
+    proxy.model.matrix <- model.matrix(proxy_formula, df)
+    proxy.variable <- all.vars(proxy_formula)[1]
+
+    df.proxy.obs <- model.frame(proxy_formula,df)
+    proxy.obs <- with(df.proxy.obs, eval(parse(text=proxy.variable)))
+
+    response.var <- all.vars(outcome_formula)[1]
+    y.obs <- with(df.obs,eval(parse(text=response.var)))
+    outcome.model.matrix <- model.matrix(outcome_formula, df.obs)
+
+    df.unobs <- df[is.na(df[[response.var]])]
+    df.unobs.y1 <- copy(df.unobs)
+    df.unobs.y1[[response.var]] <- 1
+    df.unobs.y0 <- copy(df.unobs)
+    df.unobs.y0[[response.var]] <- 0
+
+    outcome.model.matrix.y1 <- model.matrix(outcome_formula, df.unobs.y1)
+    proxy.model.matrix.y1 <- model.matrix(proxy_formula, df.unobs.y1)
+    proxy.model.matrix.y0 <- model.matrix(proxy_formula, df.unobs.y0)
+    proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))

 
     nll <- function(params){
 
     nll <- function(params){

-        df.obs <- model.frame(outcome_formula, df)
-        proxy.variable <- all.vars(proxy_formula)[1]
-        proxy.model.matrix <- model.matrix(proxy_formula, df)
-        response.var <- all.vars(outcome_formula)[1]
-        y.obs <- with(df.obs,eval(parse(text=response.var)))
-        outcome.model.matrix <- model.matrix(outcome_formula, df.obs)

 
         param.idx <- 1
         n.outcome.model.covars <- dim(outcome.model.matrix)[2]
 
         param.idx <- 1
         n.outcome.model.covars <- dim(outcome.model.matrix)[2]


@@ -38,12 +59,9 @@ measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='lo
             ll.y.obs[y.obs==0] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==0,]),log=TRUE,lower.tail=FALSE)
         }
 
             ll.y.obs[y.obs==0] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==0,]),log=TRUE,lower.tail=FALSE)
         }
 

-        df.obs <- model.frame(proxy_formula,df)

         n.proxy.model.covars <- dim(proxy.model.matrix)[2]
         proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
         n.proxy.model.covars <- dim(proxy.model.matrix)[2]
         proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]

-

         param.idx <- param.idx + n.proxy.model.covars
         param.idx <- param.idx + n.proxy.model.covars

-        proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))

 
         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
             ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
 
         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
             ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])


@@ -52,15 +70,8 @@ measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='lo
         }
 
         ll.obs <- sum(ll.y.obs + ll.w.obs)
         }
 
         ll.obs <- sum(ll.y.obs + ll.w.obs)

-
-        df.unobs <- df[is.na(df[[response.var]])]
-        df.unobs.y1 <- copy(df.unobs)
-        df.unobs.y1[[response.var]] <- 1
-        df.unobs.y0 <- copy(df.unobs)
-        df.unobs.y0[[response.var]] <- 0

         
         ## integrate out y
         
         ## integrate out y

-        outcome.model.matrix.y1 <- model.matrix(outcome_formula, df.unobs.y1)

 
         if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
             ll.y.unobs.1 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
 
         if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
             ll.y.unobs.1 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])


@@ -69,10 +80,6 @@ measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='lo
             ll.y.unobs.0 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE,lower.tail=FALSE)
         }
 
             ll.y.unobs.0 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE,lower.tail=FALSE)
         }
 

-        proxy.model.matrix.y1 <- model.matrix(proxy_formula, df.unobs.y1)
-        proxy.model.matrix.y0 <- model.matrix(proxy_formula, df.unobs.y0)
-        proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
-

         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
             ll.w.unobs.1 <- vector(mode='numeric',length=dim(proxy.model.matrix.y1)[1])
             ll.w.unobs.0 <- vector(mode='numeric',length=dim(proxy.model.matrix.y0)[1])
         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
             ll.w.unobs.1 <- vector(mode='numeric',length=dim(proxy.model.matrix.y1)[1])
             ll.w.unobs.0 <- vector(mode='numeric',length=dim(proxy.model.matrix.y0)[1])


@@ -98,173 +105,273 @@ measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='lo
     start <- rep(0.1,length(params))
     names(start) <- params
     
     start <- rep(0.1,length(params))
     names(start) <- params
     

-    fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+    if(method=='optim'){
+        fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+    } else {
+        quoted.names <- gsub("[\\(\\)]",'',names(start))
+        print(quoted.names)
+        text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(nll(params))}")
+
+        measerr_mle_nll <- eval(parse(text=text))
+        names(start) <- quoted.names
+        names(lower) <- quoted.names
+        fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
+    }

     return(fit)
 }
 
     return(fit)
 }
 

-## Experimental, and not necessary if errors are independent.
-measerr_irr_mle <- function(df, outcome_formula, outcome_family=gaussian(), rater_formula, proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit')){

 
 

-    ### in this scenario, the ground truth also has measurement error, but we have repeated measures for it. 
+measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){

 
 

-    ## probability of y given observed data.
-    df.obs <- df[!is.na(x.obs.1)]
+    df.obs <- model.frame(outcome_formula, df)
+    response.var <- all.vars(outcome_formula)[1]

     proxy.variable <- all.vars(proxy_formula)[1]
     proxy.variable <- all.vars(proxy_formula)[1]

-    df.x.obs.1 <- copy(df.obs)[,x:=1]
-    df.x.obs.0 <- copy(df.obs)[,x:=0]
-    y.obs <- df.obs[,y]
+    truth.variable <- all.vars(truth_formula)[1]
+    outcome.model.matrix <- model.matrix(outcome_formula, df)
+    proxy.model.matrix <- model.matrix(proxy_formula, df)
+    y.obs <- with(df.obs,eval(parse(text=response.var)))
+
+    df.proxy.obs <- model.frame(proxy_formula,df)
+    proxy.obs <- with(df.proxy.obs, eval(parse(text=proxy.variable)))
+    n.proxy.model.covars <- dim(proxy.model.matrix)[2]
+
+    df.truth.obs <- model.frame(truth_formula, df)
+    truth.obs <- with(df.truth.obs, eval(parse(text=truth.variable)))
+    truth.model.matrix <- model.matrix(truth_formula,df.truth.obs)
+    n.truth.model.covars <- dim(truth.model.matrix)[2]
+
+    df.unobs <- df[is.na(eval(parse(text=truth.variable)))]
+    df.unobs.x1 <- copy(df.unobs)
+    df.unobs.x1[,truth.variable] <- 1
+    df.unobs.x0 <- copy(df.unobs)
+    df.unobs.x0[,truth.variable] <- 0
+    outcome.unobs <- with(df.unobs, eval(parse(text=response.var)))
+            
+    outcome.model.matrix.x0 <- model.matrix(outcome_formula, df.unobs.x0)
+    outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1)
+        
+    proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.unobs.x0)
+    proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.unobs.x1)
+    proxy.unobs <- df.unobs[[proxy.variable]]

 
 

-    nll <- function(params){
-        outcome.model.matrix.x.obs.0 <- model.matrix(outcome_formula, df.x.obs.0)
-        outcome.model.matrix.x.obs.1 <- model.matrix(outcome_formula, df.x.obs.1)
+    truth.model.matrix.unobs <- model.matrix(truth_formula, df.unobs.x0)

 
 

+    measerr_mle_nll <- function(params){

         param.idx <- 1
         param.idx <- 1

-        n.outcome.model.covars <- dim(outcome.model.matrix.x.obs.0)[2]
+        n.outcome.model.covars <- dim(outcome.model.matrix)[2]

         outcome.params <- params[param.idx:n.outcome.model.covars]
         param.idx <- param.idx + n.outcome.model.covars
 
         outcome.params <- params[param.idx:n.outcome.model.covars]
         param.idx <- param.idx + n.outcome.model.covars
 

-        sigma.y <- params[param.idx]
-        param.idx <- param.idx + 1
-
-        ll.y.x.obs.0 <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix.x.obs.0),sd=sigma.y, log=TRUE)
-        ll.y.x.obs.1 <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix.x.obs.1),sd=sigma.y, log=TRUE)
-
-        ## assume that the two coders are statistically independent conditional on x
-        ll.x.obs.0.x0 <- vector(mode='numeric', length=nrow(df.obs))
-        ll.x.obs.1.x0 <- vector(mode='numeric', length=nrow(df.obs))
-        ll.x.obs.0.x1 <- vector(mode='numeric', length=nrow(df.obs))
-        ll.x.obs.1.x1 <- vector(mode='numeric', length=nrow(df.obs))
-
-        rater.model.matrix.x.obs.0 <- model.matrix(rater_formula, df.x.obs.0)
-        rater.model.matrix.x.obs.1 <- model.matrix(rater_formula, df.x.obs.1)
-
-        n.rater.model.covars <- dim(rater.model.matrix.x.obs.0)[2]
-        rater.0.params <- params[param.idx:(n.rater.model.covars + param.idx - 1)]
-        param.idx <- param.idx + n.rater.model.covars
+        ## likelihood for the fully observed data 
+        if(outcome_family$family == "gaussian"){
+            sigma.y <- params[param.idx]
+            param.idx <- param.idx + 1
+                                        #  outcome_formula likelihood using linear regression
+            ll.y.obs <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix),sd=sigma.y, log=TRUE)
+        } else if( (outcome_family$family == "binomial") & (outcome_family$link == "logit") )
+            ll.y.obs <- likelihood.logistic(outcome.params, y.obs, outcome.model.matrix)

 
 

-        rater.1.params <- params[param.idx:(n.rater.model.covars + param.idx - 1)]
-        param.idx <- param.idx + n.rater.model.covars
-        
-        # probability of rater 0 if x is 0 or 1
-        ll.x.obs.0.x0[df.obs$x.obs.0==1] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.0==1,]), log=TRUE)
-        ll.x.obs.0.x0[df.obs$x.obs.0==0] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.0==0,]), log=TRUE, lower.tail=FALSE)
-        ll.x.obs.0.x1[df.obs$x.obs.0==1] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.0==1,]), log=TRUE)
-        ll.x.obs.0.x1[df.obs$x.obs.0==0] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.0==0,]), log=TRUE, lower.tail=FALSE)
-
-        # probability of rater 1 if x is 0 or 1
-        ll.x.obs.1.x0[df.obs$x.obs.1==1] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.1==1,]), log=TRUE)
-        ll.x.obs.1.x0[df.obs$x.obs.1==0] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.1==0,]), log=TRUE, lower.tail=FALSE)
-        ll.x.obs.1.x1[df.obs$x.obs.1==1] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.1==1,]), log=TRUE)
-        ll.x.obs.1.x1[df.obs$x.obs.1==0] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.1==0,]), log=TRUE, lower.tail=FALSE)
-
-        proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.x.obs.0)
-        proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.x.obs.1)
-
-        n.proxy.model.covars <- dim(proxy.model.matrix.x0)[2]
+    

         proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
         param.idx <- param.idx + n.proxy.model.covars
 
         proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
         param.idx <- param.idx + n.proxy.model.covars
 

-        proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
-
-        if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
-            ll.w.obs.x0 <- vector(mode='numeric',length=dim(proxy.model.matrix.x0)[1])
-            ll.w.obs.x1 <- vector(mode='numeric',length=dim(proxy.model.matrix.x1)[1])
-
-                                        # proxy_formula likelihood using logistic regression
-            ll.w.obs.x0[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.obs==1,]),log=TRUE)
-            ll.w.obs.x0[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
-
-            ll.w.obs.x1[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.obs==1,]),log=TRUE)
-            ll.w.obs.x1[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
-        }
-
-        ## assume that the probability of x is a logistic regression depending on z
-        truth.model.matrix.obs <- model.matrix(truth_formula, df.obs)
-        n.truth.params <- dim(truth.model.matrix.obs)[2]
-        truth.params <- params[param.idx:(n.truth.params + param.idx - 1)]
-
-        ll.obs.x0 <- plogis(truth.params %*% t(truth.model.matrix.obs), log=TRUE)
-        ll.obs.x1 <- plogis(truth.params %*% t(truth.model.matrix.obs), log=TRUE, lower.tail=FALSE)
+        if( (proxy_family$family=="binomial") & (proxy_family$link=='logit'))
+            ll.w.obs <- likelihood.logistic(proxy.params, proxy.obs, proxy.model.matrix)

 
 

-        ll.obs <- colLogSumExps(rbind(ll.y.x.obs.0 + ll.x.obs.0.x0 + ll.x.obs.1.x0 + ll.obs.x0 + ll.w.obs.x0,
-                                      ll.y.x.obs.1 + ll.x.obs.0.x1 + ll.x.obs.1.x1 + ll.obs.x1 + ll.w.obs.x1))
+        truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]

 
 

-        ### NOW FOR THE FUN PART. Likelihood of the unobserved data.
-        ### we have to integrate out x.obs.0, x.obs.1, and x.
+        if( (truth_family$family=="binomial") & (truth_family$link=='logit'))
+            ll.x.obs <- likelihood.logistic(truth.params, truth.obs, truth.model.matrix)
+        
+                                # add the three likelihoods
+        ll.obs <- sum(ll.y.obs + ll.w.obs + ll.x.obs)

 
 

+        ## likelihood for the predicted data
+        ## integrate out the "truth" variable. 
+        
+        if(truth_family$family=='binomial'){
+            if(outcome_family$family=="gaussian"){
+                                        # likelihood of outcome
+                ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
+                ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
+            } else if( (outcome_family$family == "binomial") & (outcome_family$link == "logit") ){
+                ll.y.x1 <- likelihood.logistic(outcome.params, outcome.unobs, outcome.model.matrix.x1)
+                ll.y.x0 <- likelihood.logistic(outcome.params, outcome.unobs, outcome.model.matrix.x0)
+            }

 
 

-        ## THE OUTCOME
-        df.unobs <- df[is.na(x.obs)]
-        df.x.unobs.0 <- copy(df.unobs)[,x:=0]
-        df.x.unobs.1 <- copy(df.unobs)[,x:=1]
-        y.unobs <- df.unobs$y
+            if( (proxy_family$family=='binomial') & (proxy_family$link=='logit')){

 
 

-        outcome.model.matrix.x.unobs.0 <- model.matrix(outcome_formula, df.x.unobs.0)
-        outcome.model.matrix.x.unobs.1 <- model.matrix(outcome_formula, df.x.unobs.1)
+                ll.w.x0 <- likelihood.logistic(proxy.params, proxy.unobs, proxy.model.matrix.x0)
+                ll.w.x1 <- likelihood.logistic(proxy.params, proxy.unobs, proxy.model.matrix.x1)

 
 

-        ll.y.unobs.x0 <- dnorm(y.unobs, outcome.params %*% t(outcome.model.matrix.x.unobs.0), sd=sigma.y, log=TRUE)
-        ll.y.unobs.x1 <- dnorm(y.unobs, outcome.params %*% t(outcome.model.matrix.x.unobs.1), sd=sigma.y, log=TRUE)
+            }

 
 

-        
-        ## THE UNLABELED DATA
+            if(truth_family$link=='logit'){
+                                        # likelihood of truth
+                ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE)
+                ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE, lower.tail=FALSE)
+            }
+        }

 
 

-        
-        ## assume that the two coders are statistically independent conditional on x
-        ll.x.unobs.0.x0 <- vector(mode='numeric', length=nrow(df.unobs))
-        ll.x.unobs.1.x0 <- vector(mode='numeric', length=nrow(df.unobs))
-        ll.x.unobs.0.x1 <- vector(mode='numeric', length=nrow(df.unobs))
-        ll.x.unobs.1.x1 <- vector(mode='numeric', length=nrow(df.unobs))
-        
-        df.x.unobs.0[,x.obs := 1]
-        df.x.unobs.1[,x.obs := 1]
+        ll.x0 <- ll.y.x0 + ll.w.x0 + ll.x.x0
+        ll.x1 <- ll.y.x1 + ll.w.x1 + ll.x.x1
+        ll.unobs <- sum(colLogSumExps(rbind(ll.x0, ll.x1)))
+        return(-(ll.unobs + ll.obs))
+    }
+    
+    outcome.params <- colnames(model.matrix(outcome_formula,df))
+    lower <- rep(-Inf, length(outcome.params))

 
 

-        rater.model.matrix.x.unobs.0 <- model.matrix(rater_formula, df.x.unobs.0)
-        rater.model.matrix.x.unobs.1 <- model.matrix(rater_formula, df.x.unobs.1)
+    if(outcome_family$family=='gaussian'){
+        params <- c(outcome.params, 'sigma_y')
+        lower <- c(lower, 0.00001)
+    } else {
+        params <- outcome.params
+    }
+    
+    proxy.params <- colnames(model.matrix(proxy_formula, df))
+    params <- c(params, paste0('proxy_',proxy.params))
+    lower <- c(lower, rep(-Inf, length(proxy.params)))

 
 

-         
-        ## # probability of rater 0 if x is 0 or 1
-        ## ll.x.unobs.0.x0 <- colLogSumExps(rbind(plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE),
-        ##                                      plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE, lower.tail=TRUE)))
+    truth.params <- colnames(model.matrix(truth_formula, df))
+    params <- c(params, paste0('truth_', truth.params))
+    lower <- c(lower, rep(-Inf, length(truth.params)))
+    start <- rep(0.1,length(params))
+    names(start) <- params

 
 

-        ## ll.x.unobs.0.x1 <- colLogSumExps(rbind(plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE),
-        ##                                        plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE, lower.tail=TRUE)))
+    if(method=='optim'){
+        fit <- optim(start, fn = measerr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+    } else { # method='mle2'
+                
+        quoted.names <- gsub("[\\(\\)]",'',names(start))

 
 

-        ## # probability of rater 1 if x is 0 or 1
-        ## ll.x.unobs.1.x0 <- colLogSumExps(rbind(plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE),
-        ##                                      plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE, lower.tail=TRUE)))
+        text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(measerr_mle_nll(params))}")

 
 

-        ## ll.x.unobs.1.x1 <- colLogSumExps(rbind(plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE),
-        ##                                      plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE, lower.tail=TRUE)))
+        measerr_mle_nll_mle <- eval(parse(text=text))
+        names(start) <- quoted.names
+        names(lower) <- quoted.names
+        fit <- mle2(minuslogl=measerr_mle_nll_mle, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
+    }

 
 

+    return(fit)
+}

 
 

-        proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
-        proxy.model.matrix.x0.unobs <- model.matrix(proxy_formula, df.x.unobs.0)
-        proxy.model.matrix.x1.unobs <- model.matrix(proxy_formula, df.x.unobs.1)
+## Experimental, but probably works. 
+measerr_irr_mle <- function(df, outcome_formula, outcome_family=gaussian(), coder_formulas=list(x.obs.0~x, x.obs.1~x), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){

 
 

-        if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
-            ll.w.unobs.x0 <- vector(mode='numeric',length=dim(proxy.model.matrix.x0)[1])
-            ll.w.unobs.x1 <- vector(mode='numeric',length=dim(proxy.model.matrix.x1)[1])
+    ### in this scenario, the ground truth also has measurement error, but we have repeated measures for it. 
+    # this time we never get to observe the true X
+    outcome.model.matrix <- model.matrix(outcome_formula, df)
+    proxy.model.matrix <- model.matrix(proxy_formula, df)
+    response.var <- all.vars(outcome_formula)[1]
+    proxy.var <- all.vars(proxy_formula)[1]
+    param.var <- all.vars(truth_formula)[1]
+    truth.var<- all.vars(truth_formula)[1]
+    y <- with(df,eval(parse(text=response.var)))

 
 

+    nll <- function(params){
+        param.idx <- 1
+        n.outcome.model.covars <- dim(outcome.model.matrix)[2]
+        outcome.params <- params[param.idx:n.outcome.model.covars]
+        param.idx <- param.idx + n.outcome.model.covars

 
 

-                                        # proxy_formula likelihood using logistic regression
-            ll.w.unobs.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0.unobs[proxy.unobs==1,]),log=TRUE)
-            ll.w.unobs.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0.unobs[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)

 
 

-            ll.w.unobs.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1.unobs[proxy.unobs==1,]),log=TRUE)
-            ll.w.unobs.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1.unobs[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
+        if(outcome_family$family == "gaussian"){
+            sigma.y <- params[param.idx]
+            param.idx <- param.idx + 1

         }
 
         }
 

-        truth.model.matrix.unobs <- model.matrix(truth_formula, df.unobs)

 
 

-        ll.unobs.x0 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE)
-        ll.unobs.x1 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE, lower.tail=FALSE)
+        n.proxy.model.covars <- dim(proxy.model.matrix)[2]
+        proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
+        param.idx <- param.idx + n.proxy.model.covars
+        
+        df.temp <- copy(df)
+
+        if((truth_family$family == "binomial")
+           & (truth_family$link=='logit')){
+            integrate.grid <- expand.grid(replicate(1 + length(coder_formulas), c(0,1), simplify=FALSE))
+            ll.parts <- matrix(nrow=nrow(df),ncol=nrow(integrate.grid))
+            for(i in 1:nrow(integrate.grid)){
+                # setup the dataframe for this row
+                row <- integrate.grid[i,]
+
+                df.temp[[param.var]] <- row[[1]]
+                ci <- 2
+                for(coder_formula in coder_formulas){
+                    coder.var <- all.vars(coder_formula)[1]
+                    df.temp[[coder.var]] <- row[[ci]]
+                    ci <- ci + 1 
+                }
+                
+                outcome.model.matrix <- model.matrix(outcome_formula, df.temp)                
+                if(outcome_family$family == "gaussian"){
+                    ll.y <- dnorm(y, outcome.params %*% t(outcome.model.matrix), sd=sigma.y, log=TRUE)
+                }
+
+                if(proxy_family$family=="binomial" & (proxy_family$link=='logit')){
+                    proxy.model.matrix <- model.matrix(proxy_formula, df.temp)
+                    ll.w <- vector(mode='numeric', length=dim(proxy.model.matrix)[1])
+                    proxyvar <- with(df.temp,eval(parse(text=proxy.var)))
+                    ll.w[proxyvar==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxyvar==1,]),log=TRUE)
+                    ll.w[proxyvar==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxyvar==0,]),log=TRUE,lower.tail=FALSE)
+                }
+
+                ## probability of the coded variables
+                coder.lls <- matrix(nrow=nrow(df.temp),ncol=length(coder_formulas))
+                ci <- 1
+                for(coder_formula in coder_formulas){
+                    coder.model.matrix <- model.matrix(coder_formula, df.temp)
+                    n.coder.model.covars <- dim(coder.model.matrix)[2]
+                    coder.params <- params[param.idx:(n.coder.model.covars + param.idx - 1)]
+                    param.idx <- param.idx + n.coder.model.covars
+                    coder.var <- all.vars(coder_formula)[1]
+                    x.obs <- with(df.temp, eval(parse(text=coder.var)))
+                    true.codervar <- df[[all.vars(coder_formula)[1]]]
+
+                    ll.coder <- vector(mode='numeric', length=dim(coder.model.matrix)[1])
+                    ll.coder[x.obs==1] <- plogis(coder.params %*% t(coder.model.matrix[x.obs==1,]),log=TRUE)
+                    ll.coder[x.obs==0] <- plogis(coder.params %*% t(coder.model.matrix[x.obs==0,]),log=TRUE,lower.tail=FALSE)
+
+                    # don't count when we know the observed value, unless we're accounting for observed value
+                    ll.coder[(!is.na(true.codervar)) & (true.codervar != x.obs)] <- NA
+                    coder.lls[,ci] <- ll.coder
+                    ci <- ci + 1
+                }
+                
+                truth.model.matrix <- model.matrix(truth_formula, df.temp)
+                n.truth.model.covars <- dim(truth.model.matrix)[2]
+                truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]
+
+                for(coder_formula in coder_formulas){
+                    coder.model.matrix <- model.matrix(coder_formula, df.temp)
+                    n.coder.model.covars <- dim(coder.model.matrix)[2]
+                    param.idx <- param.idx - n.coder.model.covars
+                }
+
+                x <- with(df.temp, eval(parse(text=truth.var)))
+                ll.truth <- vector(mode='numeric', length=dim(truth.model.matrix)[1])
+                ll.truth[x==1] <- plogis(truth.params %*% t(truth.model.matrix[x==1,]), log=TRUE)
+                ll.truth[x==0] <- plogis(truth.params %*% t(truth.model.matrix[x==0,]), log=TRUE, lower.tail=FALSE)
+
+                true.truthvar <- df[[all.vars(truth_formula)[1]]]
+                
+                if(!is.null(true.truthvar)){
+                                        # ll.truth[(!is.na(true.truthvar)) & (true.truthvar != truthvar)] <- -Inf
+                    # ll.truth[(!is.na(true.truthvar)) & (true.truthvar == truthvar)] <- 0
+                }
+                ll.parts[,i] <- ll.y + ll.w + apply(coder.lls,1,sum) + ll.truth
+                
+            }

 
 

-        ll.unobs <- colLogSumExps(rbind(ll.unobs.x0 + ll.w.unobs.x0 + ll.y.unobs.x0,
-                                        ll.unobs.x1 + ll.w.unobs.x1 + ll.y.unobs.x1))
+            lls <- rowLogSumExps(ll.parts,na.rm=TRUE)

 
 

-        return(-1 *( sum(ll.obs) + sum(ll.unobs)))
+        ## likelihood of observed data 
+            target <- -1 * sum(lls)
+            return(target)
+        }

     }
     }

-
+        

     outcome.params <- colnames(model.matrix(outcome_formula,df))
     lower <- rep(-Inf, length(outcome.params))
 
     outcome.params <- colnames(model.matrix(outcome_formula,df))
     lower <- rep(-Inf, length(outcome.params))
 


@@ -274,139 +381,157 @@ measerr_irr_mle <- function(df, outcome_formula, outcome_family=gaussian(), rate
     } else {
         params <- outcome.params
     }
     } else {
         params <- outcome.params
     }

-    
-    rater.0.params <- colnames(model.matrix(rater_formula,df))
-    params <- c(params, paste0('rater_0',rater.0.params))
-    lower <- c(lower, rep(-Inf, length(rater.0.params)))
-
-    rater.1.params <- colnames(model.matrix(rater_formula,df))
-    params <- c(params, paste0('rater_1',rater.1.params))
-    lower <- c(lower, rep(-Inf, length(rater.1.params)))

 
     proxy.params <- colnames(model.matrix(proxy_formula, df))
     params <- c(params, paste0('proxy_',proxy.params))
 
     proxy.params <- colnames(model.matrix(proxy_formula, df))
     params <- c(params, paste0('proxy_',proxy.params))

+    positive.params <- paste0('proxy_',truth.var)

     lower <- c(lower, rep(-Inf, length(proxy.params)))
     lower <- c(lower, rep(-Inf, length(proxy.params)))

+    names(lower) <- params
+    lower[positive.params] <- 0.01
+    ci <- 0
+    
+    for(coder_formula in coder_formulas){
+        coder.params <- colnames(model.matrix(coder_formula,df))
+        params <- c(params, paste0('coder_',ci,coder.params))
+        positive.params <- paste0('coder_', ci, truth.var)
+        ci <- ci + 1
+        lower <- c(lower, rep(-Inf, length(coder.params)))
+        names(lower) <- params
+        lower[positive.params] <- 0.01
+    }

 
     truth.params <- colnames(model.matrix(truth_formula, df))
     params <- c(params, paste0('truth_', truth.params))
     lower <- c(lower, rep(-Inf, length(truth.params)))
     start <- rep(0.1,length(params))
     names(start) <- params
 
     truth.params <- colnames(model.matrix(truth_formula, df))
     params <- c(params, paste0('truth_', truth.params))
     lower <- c(lower, rep(-Inf, length(truth.params)))
     start <- rep(0.1,length(params))
     names(start) <- params

+    names(lower) <- params

     
     

-    fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+    if(method=='optim'){
+        print(start)
+        fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+    } else {
+                
+        quoted.names <- gsub("[\\(\\)]",'',names(start))
+        print(quoted.names)
+        text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(nll(params))}")
+
+        measerr_mle_nll <- eval(parse(text=text))
+        names(start) <- quoted.names
+        names(lower) <- quoted.names
+        fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, method='L-BFGS-B',control=list(maxit=1e6))
+    }
+

     return(fit)
 }
 
     return(fit)
 }
 

+## Experimental, and does not work.
+measerr_irr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), coder_formulas=list(y.obs.0~y+w_pred+y.obs.1,y.obs.1~y+w_pred+y.obs.0), proxy_formula=w_pred~y, proxy_family=binomial(link='logit'),method='optim'){
+    integrate.grid <- expand.grid(replicate(1 + length(coder_formulas), c(0,1), simplify=FALSE))
+#    print(integrate.grid)

 
 

-measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit')){

 
 

-    measrr_mle_nll <- function(params){
-        df.obs <- model.frame(outcome_formula, df)
-        proxy.variable <- all.vars(proxy_formula)[1]
-        proxy.model.matrix <- model.matrix(proxy_formula, df)
-        response.var <- all.vars(outcome_formula)[1]
-        y.obs <- with(df.obs,eval(parse(text=response.var)))
+    outcome.model.matrix <- model.matrix(outcome_formula, df)
+    n.outcome.model.covars <- dim(outcome.model.matrix)[2]

 
 

-        outcome.model.matrix <- model.matrix(outcome_formula, df)

 
 

+    ### in this scenario, the ground truth also has measurement error, but we have repeated measures for it. 
+    # this time we never get to observe the true X
+    nll <- function(params){

         param.idx <- 1
         param.idx <- 1

-        n.outcome.model.covars <- dim(outcome.model.matrix)[2]

         outcome.params <- params[param.idx:n.outcome.model.covars]
         param.idx <- param.idx + n.outcome.model.covars
         outcome.params <- params[param.idx:n.outcome.model.covars]
         param.idx <- param.idx + n.outcome.model.covars

+        proxy.model.matrix <- model.matrix(proxy_formula, df)
+        n.proxy.model.covars <- dim(proxy.model.matrix)[2]
+        response.var <- all.vars(outcome_formula)[1]

 
 

-        ## likelihood for the fully observed data 

         if(outcome_family$family == "gaussian"){
             sigma.y <- params[param.idx]
             param.idx <- param.idx + 1
         if(outcome_family$family == "gaussian"){
             sigma.y <- params[param.idx]
             param.idx <- param.idx + 1

-
-                                        #  outcome_formula likelihood using linear regression
-            ll.y.obs <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix),sd=sigma.y, log=TRUE)

         }
         }

-        
-        df.obs <- model.frame(proxy_formula,df)
-        n.proxy.model.covars <- dim(proxy.model.matrix)[2]
+

         proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
         param.idx <- param.idx + n.proxy.model.covars
         proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
         param.idx <- param.idx + n.proxy.model.covars

-        proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
-
-        if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
-            ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])

 
 

-                                        # proxy_formula likelihood using logistic regression
-            ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
-            ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
-        }
-
-        df.obs <- model.frame(truth_formula, df)
-        truth.variable <- all.vars(truth_formula)[1]
-        truth.obs <- with(df.obs, eval(parse(text=truth.variable)))
-        truth.model.matrix <- model.matrix(truth_formula,df)
-        n.truth.model.covars <- dim(truth.model.matrix)[2]
-        
-        truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]
-
-        if( (truth_family$family=="binomial") & (truth_family$link=='logit')){
-            ll.x.obs <- vector(mode='numeric',length=dim(truth.model.matrix)[1])
-
-                                        # truth_formula likelihood using logistic regression
-            ll.x.obs[truth.obs==1] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==1,]),log=TRUE)
-            ll.x.obs[truth.obs==0] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==0,]),log=TRUE, lower.tail=FALSE)
-        }
-        
-                                        # add the three likelihoods
-        ll.obs <- sum(ll.y.obs + ll.w.obs + ll.x.obs)
-
-        ## likelihood for the predicted data
-        ## integrate out the "truth" variable. 
-        
-        if(truth_family$family=='binomial'){
-            df.unobs <- df[is.na(eval(parse(text=truth.variable)))]
-            df.unobs.x1 <- copy(df.unobs)
-            df.unobs.x1[,'x'] <- 1
-            df.unobs.x0 <- copy(df.unobs)
-            df.unobs.x0[,'x'] <- 0
-            outcome.unobs <- with(df.unobs, eval(parse(text=response.var)))
-            
-            outcome.model.matrix.x0 <- model.matrix(outcome_formula, df.unobs.x0)
-            outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1)
-            if(outcome_family$family=="gaussian"){
-
-                                        # likelihood of outcome
-                ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
-                ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
+        df.temp <- copy(df)
+
+        if((outcome_family$family == "binomial")
+           & (outcome_family$link=='logit')){
+            ll.parts <- matrix(nrow=nrow(df),ncol=nrow(integrate.grid))
+            for(i in 1:nrow(integrate.grid)){
+                # setup the dataframe for this row
+                row <- integrate.grid[i,]
+
+                df.temp[[response.var]] <- row[[1]]
+                ci <- 2
+                for(coder_formula in coder_formulas){
+                    codervar <- all.vars(coder_formula)[1]
+                    df.temp[[codervar]] <- row[[ci]]
+                    ci <- ci + 1 
+                }
+                
+                outcome.model.matrix <- model.matrix(outcome_formula, df.temp)                
+                if(outcome_family$family == "gaussian"){
+                    ll.y <- dnorm(df.temp[[response.var]], outcome.params %*% t(outcome.model.matrix), sd=sigma.y, log=T)
+                }
+
+                if(outcome_family$family == "binomial" & (outcome_family$link=='logit')){
+                    ll.y <- vector(mode='numeric',length=nrow(df.temp))
+                    ll.y[df.temp[[response.var]]==1] <- plogis( outcome.params %*% t(outcome.model.matrix), log=TRUE)
+                    ll.y[df.temp[[response.var]]==0] <- plogis( outcome.params %*% t(outcome.model.matrix), log=TRUE,lower.tail=FALSE)
+                }
+
+                if(proxy_family$family=="binomial" & (proxy_family$link=='logit')){
+                    proxy.model.matrix <- model.matrix(proxy_formula, df.temp)
+                    ll.w <- vector(mode='numeric', length=dim(proxy.model.matrix)[1])
+                    proxyvar <- with(df.temp,eval(parse(text=all.vars(proxy_formula)[1])))
+                    ll.w[proxyvar==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxyvar==1,]),log=TRUE)
+                    ll.w[proxyvar==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxyvar==0,]),log=TRUE,lower.tail=FALSE)
+                }
+
+                ## probability of the coded variables
+                coder.lls <- matrix(nrow=nrow(df.temp),ncol=length(coder_formulas))
+                ci <- 1
+                for(coder_formula in coder_formulas){
+                    coder.model.matrix <- model.matrix(coder_formula, df.temp)
+                    n.coder.model.covars <- dim(coder.model.matrix)[2]
+                    coder.params <- params[param.idx:(n.coder.model.covars + param.idx - 1)]
+                    param.idx <- param.idx + n.coder.model.covars
+                    codervar <- with(df.temp, eval(parse(text=all.vars(coder_formula)[1])))
+                    true.codervar <- df[[all.vars(coder_formula)[1]]]
+
+                    ll.coder <- vector(mode='numeric', length=dim(coder.model.matrix)[1])
+                    ll.coder[codervar==1] <- plogis(coder.params %*% t(coder.model.matrix[codervar==1,]),log=TRUE)
+                    ll.coder[codervar==0] <- plogis(coder.params %*% t(coder.model.matrix[codervar==0,]),log=TRUE,lower.tail=FALSE)
+
+                    # don't count when we know the observed value, unless we're accounting for observed value
+                    ll.coder[(!is.na(true.codervar)) & (true.codervar != codervar)] <- NA
+                    coder.lls[,ci] <- ll.coder
+                    ci <- ci + 1
+                }
+
+                for(coder_formula in coder_formulas){
+                    coder.model.matrix <- model.matrix(coder_formula, df.temp)
+                    n.coder.model.covars <- dim(coder.model.matrix)[2]
+                    param.idx <- param.idx - n.coder.model.covars
+                }
+
+                ll.parts[,i] <- ll.y + ll.w + apply(coder.lls,1,function(x) sum(x)) 
+                

             }
 
             }
 

-            if( (proxy_family$family=='binomial') & (proxy_family$link=='logit')){
-
-                proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.unobs.x0)
-                proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.unobs.x1)
-                proxy.unobs <- df.unobs[[proxy.variable]]
-                ll.w.x0 <- vector(mode='numeric', length=dim(df.unobs)[1])
-                ll.w.x1 <- vector(mode='numeric', length=dim(df.unobs)[1])
-
-                                        # likelihood of proxy
-                ll.w.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==1,]), log=TRUE)
-                ll.w.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==1,]), log=TRUE)
-
-                ll.w.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
-                ll.w.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
-            }
+            lls <- rowLogSumExps(ll.parts,na.rm=TRUE)

 
 

-            if(truth_family$link=='logit'){
-                truth.model.matrix <- model.matrix(truth_formula, df.unobs.x0)
-                                        # likelihood of truth
-                ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE)
-                ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE, lower.tail=FALSE)
-            }
+            ## likelihood of observed data 
+            target <- -1 * sum(lls)
+#            print(target)
+#            print(params)
+            return(target)

         }
         }

-
-        ll.x0 <- ll.y.x0 + ll.w.x0 + ll.x.x0
-        ll.x1 <- ll.y.x1 + ll.w.x1 + ll.x.x1
-        ll.unobs <- sum(colLogSumExps(rbind(ll.x0, ll.x1)))
-        return(-(ll.unobs + ll.obs))

     }
     }

-    
+        

     outcome.params <- colnames(model.matrix(outcome_formula,df))
     outcome.params <- colnames(model.matrix(outcome_formula,df))

+    response.var <- all.vars(outcome_formula)[1]

     lower <- rep(-Inf, length(outcome.params))
 
     if(outcome_family$family=='gaussian'){
     lower <- rep(-Inf, length(outcome.params))
 
     if(outcome_family$family=='gaussian'){


@@ -415,18 +540,54 @@ measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_fo
     } else {
         params <- outcome.params
     }
     } else {
         params <- outcome.params
     }

-    
+
+    ## constrain the model of the coder and proxy vars
+    ## this is to ensure identifiability
+    ## it is a safe assumption because the coders aren't hostile (wrong more often than right)
+    ## so we can assume that y ~Bw, B is positive

     proxy.params <- colnames(model.matrix(proxy_formula, df))
     proxy.params <- colnames(model.matrix(proxy_formula, df))

+    positive.params <- paste0('proxy_',response.var)

     params <- c(params, paste0('proxy_',proxy.params))
     lower <- c(lower, rep(-Inf, length(proxy.params)))
     params <- c(params, paste0('proxy_',proxy.params))
     lower <- c(lower, rep(-Inf, length(proxy.params)))

+    names(lower) <- params
+    lower[positive.params] <- 0.001
+
+    ci <- 0
+    for(coder_formula in coder_formulas){
+        coder.params <- colnames(model.matrix(coder_formula,df))
+        latent.coder.params <- coder.params %in% response.var
+        params <- c(params, paste0('coder_',ci,coder.params))
+        positive.params <- paste0('coder_',ci,response.var)
+        ci <- ci + 1
+        lower <- c(lower, rep(-Inf, length(coder.params)))
+        names(lower) <-params
+        lower[positive.params] <- 0.001
+    }

 
 

-    truth.params <- colnames(model.matrix(truth_formula, df))
-    params <- c(params, paste0('truth_', truth.params))
-    lower <- c(lower, rep(-Inf, length(truth.params)))
-    start <- rep(0.1,length(params))
-    names(start) <- params
+    ## init by using the "loco model"
+    temp.df <- copy(df)
+    temp.df <- temp.df[y.obs.1 == y.obs.0, y:=y.obs.1]
+    loco.model <- glm(outcome_formula, temp.df, family=outcome_family)

     
     

-    fit <- optim(start, fn = measrr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+    start <- rep(1,length(params))
+    names(start) <- params
+    start[names(coef(loco.model))] <- coef(loco.model)
+    names(lower) <- params
+    if(method=='optim'){
+        print(lower)
+        fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE,control=list(maxit=1e6))
+    } else {
+                
+        quoted.names <- gsub("[\\(\\)]",'',names(start))
+        print(quoted.names)
+        text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(nll(params))}")
+
+        measerr_mle_nll <- eval(parse(text=text))
+        names(start) <- quoted.names
+        names(lower) <- quoted.names
+        fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
+    }

 
     return(fit)
 }
 
     return(fit)
 }

+