##############################################################################
# S A n D M a n scripts to source data from ODK forms and update media files #
##############################################################################

#library(devtools)
#install_github("onaio/ona.R")
library(ona)
library(RCurl)
library(httr)
library(RJSONIO)
library(stringr)
library(plyr)
library(lubridate)
library(sp)
library(doBy)

source("C:/Users/User/Documents/ACAI/SAnDMan/ODK_functions.R")

#source("D://workspace//ACAI//ODK_functions.R")

setwd("C:/Users/User/Documents/ACAI/SAnDMan/SAnDMan_forms")
forms <- c("Register_PC", "Register_EX", "Assign_FDTLPO")
target <-  "C:/Users/User/Documents/ACAI/SAnDMan/ACAI_dwnld_briefCase"
briefCaseDwnld(forms=forms$form_id, target=target, source="ONA")
setwd(target)


#Reading in registered projects and experiments
dsPC <- dropGroupNames(read.csv("Register_PC.csv", stringsAsFactors = FALSE, na.strings=c(""," ","NA")))
dsPC$project <- ifelse(is.na(dsPC$project), dsPC$project_abbr, dsPC$project)

dsEX1 <- dropGroupNames(read.csv("Register_EX.csv", stringsAsFactors = FALSE, na.strings=c(""," ","NA")))
dsEX2 <- dropGroupNames(read.csv("Register_EX-treatmentRepeat.csv", stringsAsFactors = FALSE, na.strings=c(""," ","NA")))
dsEX  <- mergeODKforms(dsEX1, dsEX2)

dsAS1 <- dropGroupNames(read.csv("Assign_FDTLPO.csv", stringsAsFactors = FALSE, na.strings=c(""," ","NA")))
dsAS2 <- dropGroupNames(read.csv("Assign_FDTLPO-trial.csv", stringsAsFactors = FALSE, na.strings=c(""," ","NA")))
dsAS3 <- dropGroupNames(read.csv("Assign_FDTLPO-plot.csv", stringsAsFactors = FALSE, na.strings=c(""," ","NA")))

setwd("D:/03-projects/ACAI/ODK briefcase storage/created forms/SandMan/media files")


#creating projects.csv for ODK to select existing projects and projectCodes
projects <- unique(na.omit(subset(dsPC, select=c(project, projectCode))))
projects$project_label <- projects$project
write.csv(projects, "projects.csv", row.names = FALSE)


#creating countries.csv for ODK to select existing countries for each project
countries <- strsplit(dsPC$country, split = " ")
countries <- data.frame(project = rep(dsPC$project, sapply(countries, length)), 
                        country = unlist(countries))
countries <- merge(countries, unique(na.omit(subset(dsPC, select=c(project, projectCode)))))
countries <- unique(countries)
countries <- merge(countries, read.csv("africancountries.csv"))
countries <- countries[order(countries$projectCode),]
write.csv(countries, "countries.csv", row.names = FALSE)


#creating experiments.csv to select existing experiments for each project
experiments <- subset(dsEX1, select=c(project, projectCode, expName, expFamily, expVersion, expCode, expCode_label, nrTreats))
experiments <- unique(na.omit(experiments))
write.csv(experiments, "experiments.csv", row.names = FALSE)


#creating experiment_countries.csv to select existing experiments by country
experiments_countries <- subset(dsEX1, select=c(expCode, expCode_copy, country))
experiments_countries$expCode <- ifelse(is.na(experiments_countries$expCode), experiments_countries$expCode_copy, experiments_countries$expCode)
experiments_countries <- subset(experiments_countries, select=-expCode_copy)
experiments_countries <- merge(experiments_countries, experiments)
experiments_countries <- unique(na.omit(experiments_countries))
write.csv(experiments_countries, "experiments_countries.csv", row.names = FALSE)


#creating treatments.csv to obtain list of treatments by expCode
treatments <- subset(dsEX, select=c(expCode, treatNr, treatCode_label, treatName, treatCode))
treatments$expCode_treatNr <- paste0(treatments$expCode, "_", treatments$treatNr)
treatments <- unique(na.omit(treatments))
write.csv(treatments, "treatments.csv", row.names = FALSE)


#creating fields.csv
fields <- subset(dsAS1, select=c(FDID2_new, FD_name_new, projectCode, country, lonr, latr))
fields <- unique(na.omit(fields))
fields <- fields[rep(seq_len(nrow(fields)), each=9),]
fields$lonr <- fields$lonr + rep(c(-0.005, 0, 0.005), 3)
fields$latr <- fields$latr + rep(c(-0.005, 0, 0.005), each=3)
fields$lookup <- paste0(fields$projectCode, fields$country, "_E", fields$lonr, "N", fields$latr)
fields <- subset(fields, select = -c(projectCode, country, lonr, latr))
fields <- data.frame(FDID2 = fields$FDID2_new,
                     FDID2_label = fields$FD_name_new,
                     lookup = fields$lookup)
write.csv(fields, "fields.csv", row.names = FALSE)


#creating trials.csv
trials <- subset(mergeODKforms(dsAS1, dsAS2), select=c(TLID2_new, TL_name_new, FDID2_new, FDID2))
trials$FDID2 <- ifelse(is.na(trials$FDID2), trials$FDID2_new, trials$FDID2)
trials <- subset(trials, select=-FDID2_new)
trials <- unique(na.omit(trials))
trials <- trials[order(trials$TL_name_new),]
names(trials)[names(trials)=="TLID2_new"] <- "TLID2"
names(trials)[names(trials)=="TL_name_new"] <- "TLID2_label"
write.csv(trials, "trials.csv", row.names = FALSE)


#creating plots.csv
plots <- subset(mergeODKforms(dsAS1, dsAS2, dsAS3), select=c(POID2_new, plotName, TLID2_new, TLID2))
plots$TLID2 <- ifelse(is.na(plots$TLID2), plots$TLID2_new, plots$TLID2)
plots <- subset(plots, select=-TLID2_new)
plots <- unique(na.omit(plots))
names(plots)[names(plots)=="POID2_new"] <- "POID2"
names(plots)[names(plots)=="plotName"] <- "POID2_label"
write.csv(plots, "plots.csv", row.names = FALSE)


# Get tokens from ona
up <- scan(paste(target, "/", "pws.txt", sep=""), what="character")
raw.result <- GET("https://api.ona.io/api/v1/user.json", authenticate(user = up[1], password = up[2]))
raw.result.char<-rawToChar(raw.result$content)
raw.result.json<-fromJSON(raw.result.char)
TOKEN_KEY <- raw.result.json$temp_token

# Delete and upload media files
for (i in forms$xform_id){
  
  media.files <- names(forms)[which(forms[forms$xform_id==i, 1:ncol(forms)]==1)]
  
  for (j in media.files){
    
    #meta_ID = readChar(paste0(j, ".txt"), 10)
    #DELETE(paste("https://api.ona.io/api/v1/metadata/", meta_ID), add_headers(Authorization=paste("Temptoken ", TOKEN_KEY)))

    new_id = ''
    while(new_id ==''){
      # upload csv
      header=c(Authorization=paste("Temptoken ", TOKEN_KEY), `Content-Type` = 'multipart/form-data')
      post.user.results <- postForm("https://api.ona.io/api/v1/metadata.json",
                                    data_value = paste0(j, ".csv"),
                                    data_type = 'media',
                                    xform = i,
                                    data_file = fileUpload(filename = paste0(j, ".csv"), contentType = 'text/csv'), .opts=list(httpheader=header), 
                                    verbose = TRUE)
      # get user ID
      plots.raw.result.json <- fromJSON(post.user.results)
      new_id <- plots.raw.result.json$id
    }
    
    cat(new_id, file = paste0(j, ".txt"))
    print(paste0("media file ", j, " with ID ", new_id, " succesfully uploaded for xform ID ", i))

  }

}