+
+ if(method=='optim'){
+ fit <- optim(start, fn = measerr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+ } else { # method='mle2'
+
+ quoted.names <- gsub("[\\(\\)]",'',names(start))
+
+ text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(measerr_mle_nll(params))}")
+
+ measerr_mle_nll_mle <- eval(parse(text=text))
+ names(start) <- quoted.names
+ names(lower) <- quoted.names
+ fit <- mle2(minuslogl=measerr_mle_nll_mle, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
+ }
+
+ return(fit)
+}
+
+## Experimental, but probably works.
+measerr_irr_mle <- function(df, outcome_formula, outcome_family=gaussian(), coder_formulas=list(x.obs.0~x, x.obs.1~x), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){
+
+ ### in this scenario, the ground truth also has measurement error, but we have repeated measures for it.
+ # this time we never get to observe the true X
+ outcome.model.matrix <- model.matrix(outcome_formula, df)
+ proxy.model.matrix <- model.matrix(proxy_formula, df)
+ response.var <- all.vars(outcome_formula)[1]
+ proxy.var <- all.vars(proxy_formula)[1]
+ param.var <- all.vars(truth_formula)[1]
+ truth.var<- all.vars(truth_formula)[1]
+ y <- with(df,eval(parse(text=response.var)))
+
+ nll <- function(params){
+ param.idx <- 1
+ n.outcome.model.covars <- dim(outcome.model.matrix)[2]
+ outcome.params <- params[param.idx:n.outcome.model.covars]
+ param.idx <- param.idx + n.outcome.model.covars
+
+
+ if(outcome_family$family == "gaussian"){
+ sigma.y <- params[param.idx]
+ param.idx <- param.idx + 1
+ }
+
+
+ n.proxy.model.covars <- dim(proxy.model.matrix)[2]
+ proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
+ param.idx <- param.idx + n.proxy.model.covars
+
+ df.temp <- copy(df)
+
+ if((truth_family$family == "binomial")
+ & (truth_family$link=='logit')){
+ integrate.grid <- expand.grid(replicate(1 + length(coder_formulas), c(0,1), simplify=FALSE))
+ ll.parts <- matrix(nrow=nrow(df),ncol=nrow(integrate.grid))
+ for(i in 1:nrow(integrate.grid)){
+ # setup the dataframe for this row
+ row <- integrate.grid[i,]
+
+ df.temp[[param.var]] <- row[[1]]
+ ci <- 2
+ for(coder_formula in coder_formulas){
+ coder.var <- all.vars(coder_formula)[1]
+ df.temp[[coder.var]] <- row[[ci]]
+ ci <- ci + 1
+ }
+
+ outcome.model.matrix <- model.matrix(outcome_formula, df.temp)
+ if(outcome_family$family == "gaussian"){
+ ll.y <- dnorm(y, outcome.params %*% t(outcome.model.matrix), sd=sigma.y, log=TRUE)
+ }
+
+ if(proxy_family$family=="binomial" & (proxy_family$link=='logit')){
+ proxy.model.matrix <- model.matrix(proxy_formula, df.temp)
+ ll.w <- vector(mode='numeric', length=dim(proxy.model.matrix)[1])
+ proxyvar <- with(df.temp,eval(parse(text=proxy.var)))
+ ll.w[proxyvar==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxyvar==1,]),log=TRUE)
+ ll.w[proxyvar==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxyvar==0,]),log=TRUE,lower.tail=FALSE)
+ }
+
+ ## probability of the coded variables
+ coder.lls <- matrix(nrow=nrow(df.temp),ncol=length(coder_formulas))
+ ci <- 1
+ for(coder_formula in coder_formulas){
+ coder.model.matrix <- model.matrix(coder_formula, df.temp)
+ n.coder.model.covars <- dim(coder.model.matrix)[2]
+ coder.params <- params[param.idx:(n.coder.model.covars + param.idx - 1)]
+ param.idx <- param.idx + n.coder.model.covars
+ coder.var <- all.vars(coder_formula)[1]
+ x.obs <- with(df.temp, eval(parse(text=coder.var)))
+ true.codervar <- df[[all.vars(coder_formula)[1]]]
+
+ ll.coder <- vector(mode='numeric', length=dim(coder.model.matrix)[1])
+ ll.coder[x.obs==1] <- plogis(coder.params %*% t(coder.model.matrix[x.obs==1,]),log=TRUE)
+ ll.coder[x.obs==0] <- plogis(coder.params %*% t(coder.model.matrix[x.obs==0,]),log=TRUE,lower.tail=FALSE)
+
+ # don't count when we know the observed value, unless we're accounting for observed value
+ ll.coder[(!is.na(true.codervar)) & (true.codervar != x.obs)] <- NA
+ coder.lls[,ci] <- ll.coder
+ ci <- ci + 1
+ }
+
+ truth.model.matrix <- model.matrix(truth_formula, df.temp)
+ n.truth.model.covars <- dim(truth.model.matrix)[2]
+ truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]
+
+ for(coder_formula in coder_formulas){
+ coder.model.matrix <- model.matrix(coder_formula, df.temp)
+ n.coder.model.covars <- dim(coder.model.matrix)[2]
+ param.idx <- param.idx - n.coder.model.covars
+ }
+
+ x <- with(df.temp, eval(parse(text=truth.var)))
+ ll.truth <- vector(mode='numeric', length=dim(truth.model.matrix)[1])
+ ll.truth[x==1] <- plogis(truth.params %*% t(truth.model.matrix[x==1,]), log=TRUE)
+ ll.truth[x==0] <- plogis(truth.params %*% t(truth.model.matrix[x==0,]), log=TRUE, lower.tail=FALSE)
+
+ true.truthvar <- df[[all.vars(truth_formula)[1]]]
+
+ if(!is.null(true.truthvar)){
+ # ll.truth[(!is.na(true.truthvar)) & (true.truthvar != truthvar)] <- -Inf
+ # ll.truth[(!is.na(true.truthvar)) & (true.truthvar == truthvar)] <- 0
+ }
+ ll.parts[,i] <- ll.y + ll.w + apply(coder.lls,1,sum) + ll.truth
+
+ }
+
+ lls <- rowLogSumExps(ll.parts,na.rm=TRUE)
+
+ ## likelihood of observed data
+ target <- -1 * sum(lls)
+ return(target)
+ }
+ }
+
+ outcome.params <- colnames(model.matrix(outcome_formula,df))
+ lower <- rep(-Inf, length(outcome.params))
+
+ if(outcome_family$family=='gaussian'){
+ params <- c(outcome.params, 'sigma_y')
+ lower <- c(lower, 0.00001)
+ } else {
+ params <- outcome.params
+ }
+
+ proxy.params <- colnames(model.matrix(proxy_formula, df))
+ params <- c(params, paste0('proxy_',proxy.params))
+ positive.params <- paste0('proxy_',truth.var)
+ lower <- c(lower, rep(-Inf, length(proxy.params)))
+ names(lower) <- params
+ lower[positive.params] <- 0.01
+ ci <- 0
+
+ for(coder_formula in coder_formulas){
+ coder.params <- colnames(model.matrix(coder_formula,df))
+ params <- c(params, paste0('coder_',ci,coder.params))
+ positive.params <- paste0('coder_', ci, truth.var)
+ ci <- ci + 1
+ lower <- c(lower, rep(-Inf, length(coder.params)))
+ names(lower) <- params
+ lower[positive.params] <- 0.01
+ }
+
+ truth.params <- colnames(model.matrix(truth_formula, df))
+ params <- c(params, paste0('truth_', truth.params))
+ lower <- c(lower, rep(-Inf, length(truth.params)))
+ start <- rep(0.1,length(params))
+ names(start) <- params
+ names(lower) <- params
projects / ml_measurement_error_public.git / blobdiff