diff --git a/DESCRIPTION b/DESCRIPTION index 5d0afae..9e3b0a0 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -14,8 +14,8 @@ Imports: tcltk, gWidgets2, gWidgets2tcltk, - ROracle, DBI, netmensuration, lubridate, - stringr + stringr, + RODBC \ No newline at end of file diff --git a/NAMESPACE b/NAMESPACE index ea94497..7dba9a9 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -6,10 +6,10 @@ export(get.oracle.table) export(ilts.format.merge) export(init.project.vars) import(DBI) -import(ROracle) import(gWidgets2) import(gWidgets2tcltk) import(lubridate) import(netmensuration) import(stringr) import(tcltk) +import(RODBC) diff --git a/R/ilts.operations.R b/R/ilts.operations.R index 16ca604..aeb3d68 100644 --- a/R/ilts.operations.R +++ b/R/ilts.operations.R @@ -4,6 +4,18 @@ assign('manual.archive', "", pkg.env) assign('oracle.server', "", pkg.env) assign('oracle.user', "", pkg.env) assign('oracle.password', "", pkg.env) +options(stringsAsFactors = F) + +#' @title rename.df +#' @description renaming columns +#' @export +rename.df = function(x, n0, n1) { + if(!length(n0)== length(n1)) stop('length of names and renames need to be the same length') + for(i in 1:length(n0)){ + names(x)[which(names(x)==n0[i])] = n1[i] + } + return(x) +} #' @title init.project.vars #' @description Set global database variables by user account @@ -12,12 +24,13 @@ assign('oracle.password', "", pkg.env) init.project.vars = function() { if(exists("bio.datadirectory.ilts")){ assign('manual.archive', bio.datadirectory.ilts , pkg.env) - } - else{ + } + if(!exists("bio.datadirectory.ilts")){ + print('A window will open that you need to choose the working folder.') assign('manual.archive', gWidgets2::gfile(text = "Select project work directory", type = "selectdir"), pkg.env) - - } - + } + dir.create(pkg.env$manual.archive, showWarnings = F) + #Take from snowcrab users .Rprofile.site if(exists("oracle.snowcrab.server")){ assign('oracle.server', oracle.snowcrab.server, pkg.env) @@ -47,8 +60,9 @@ init.project.vars = function() { #' @return dataframe #' @export esonar2df = function(esonar = NULL) { - names(esonar) - colnames(esonar) = c("CPUDateTime","GPSDate","GPSTime","Latitude","Longitude","Speed","Heading","Validity","TransducerName","SensorName","SensorValue","ErrorCode","Hydrophone","SignalStrength", "setno", "latedit", "trip", "datetime") + + esonar = rename.df(esonar, c( 'CPUDATEANDTIME','GPSTIME','LATITUDE','LONGITUDE','SPEED','HEADING','VALIDITY','TRANSDUCERNAME','SENSORNAME','SENSORVALUE','ERRORCODE','HYDROPHONE','SIGNALSTRENGTH','SET_NO','LATEDIT','TRIP_ID','GPSDATE','timestamp'), + c("CPUDateTime","GPSTime","Latitude","Longitude","Speed","Heading","Validity","TransducerName","SensorName","SensorValue","ErrorCode","Hydrophone","SignalStrength", "setno", "latedit", "trip", "GPSDate","datetime")) esonar$primary = NA #Headline esonar$secondary = NA #Is nothing but may need in file @@ -74,27 +88,34 @@ esonar2df = function(esonar = NULL) { esonar$Validity = NULL esonar$ErrorCode = NULL esonar$Heading = NULL - colnames(esonar) = c("Date","Time","Latitude","Longitude","Speed", "Setno", "latedit","Trip","timestamp", "Primary","Secondary","WingSpread","Roll", "Pitch") + esonar = rename.df(esonar, c('GPSTime','Latitude','Longitude','Speed','setno','latedit','trip','GPSDate','datetime','primary','secondary','wingspread','STBDRoll','STBDPitch'), + c("Time","Latitude","Longitude","Speed", "Setno", "latedit","Trip","Date","timestamp", "Primary","Secondary","WingSpread","Roll", "Pitch")) return(esonar) } -#' @title get.acoustic.releases +#' @title get.oracle.table #' @description Get data from Oracle Database table -#' @import ROracle DBI +#' @import ROracle DBI RODBC #' @param tn tablename to get #' @param oracle.user your oracle username #' @param oracle.password your oracle password #' @return dataframe #' @export -get.oracle.table = function(tn = "",server = pkg.env$oracle.server, user =pkg.env$oracle.user, password = pkg.env$oracle.password){ +get.oracle.table = function(tn = "",server = pkg.env$oracle.server, user =pkg.env$oracle.user, password = pkg.env$oracle.password, RODBC=F){ if(tn == "")stop("You must provide a tablename to 'get.oracle.table'!!") - +if(!RODBC){ drv <- dbDriver("Oracle") con <- ROracle::dbConnect(drv, username = user, password = password, dbname = server) res <- ROracle::dbSendQuery(con, paste("select * from ", tn, sep="")) res <- fetch(res) ROracle::dbDisconnect(con) +} +if(RODBC) { + drv = odbcConnect(dsn = server ,uid = user, pwd = password) + res = sqlQuery(drv,paste("select * from ", tn,";", sep="")) + odbcCloseAll() + } return(res) } @@ -107,9 +128,10 @@ get.oracle.table = function(tn = "",server = pkg.env$oracle.server, user =pkg.en #' @import netmensuration lubridate #' @return list of lists. Format (top to bottom) year-set-data #' @export -ilts.format.merge = function(update = TRUE, user = "", years = ""){ +ilts.format.merge = function(update = TRUE, user = "", years = "", use_RODBC=F, use_local=F, depth.plot=F,temperature.plot=F, sensor.file='ILTS_SENSORS_TEMP.csv', minilog.file = 'MINILOG_TEMP.csv', seabird.file = 'ILTS_TEMPERATURE.csv'){ #Set up database server, user and password init.project.vars() + options(stringsAsFactors=F) cont=TRUE if(user == "")stop("You must call this function with a user. exa. ilts.format.merge(update = TRUE, user = 'John'" ) @@ -133,12 +155,27 @@ ilts.format.merge = function(update = TRUE, user = "", years = ""){ plotdata = F #Alternative plotting that we do not require #Pull in the sensor data, this will be formatted and looped thru by trip then set. - esona = get.oracle.table(tn = "FRAILC.ILTS_SENSORS_TEMP") + dir.create(file.path(pkg.env$manual.archive,'raw'), showWarnings = F) + + if(!use_local) { esona = get.oracle.table(tn = "FRAILC.ILTS_SENSORS_TEMP", RODBC=use_RODBC) + write.csv(esona,file=file.path(pkg.env$manual.archive, 'raw', 'ILTS_SENSORS_TEMP.csv'),row.names = F) + } + if(use_local) esona = read.csv(file.path(pkg.env$manual.archive, 'raw', sensor.file)) + esona$GPSTIME[which(nchar(esona$GPSTIME)==5)] = paste("0", esona$GPSTIME[which(nchar(esona$GPSTIME)==5)], sep="") + esona$GPSTIME[which(nchar(esona$GPSTIME)==5)] = paste("0", esona$GPSTIME[which(nchar(esona$GPSTIME)==5)], sep="") + esona$GPSTIME[which(nchar(esona$GPSTIME)==4)] = paste("00", esona$GPSTIME[which(nchar(esona$GPSTIME)==4)], sep="") + esona$GPSTIME[which(nchar(esona$GPSTIME)==3)] = paste("000", esona$GPSTIME[which(nchar(esona$GPSTIME)==3)], sep="") + esona$GPSTIME[which(nchar(esona$GPSTIME)==2)] = paste("0000", esona$GPSTIME[which(nchar(esona$GPSTIME)==2)], sep="") + esona$GPSTIME[which(nchar(esona$GPSTIME)==1)] = paste("00000", esona$GPSTIME[which(nchar(esona$GPSTIME)==1)], sep="") + esona$timestamp = lubridate::ymd_hms(paste(as.character(lubridate::date(esona$GPSDATE)), esona$GPSTIME, sep=" "), tz="UTC" ) esona = esona[ order(esona$timestamp , decreasing = FALSE ),] - err = which(is.na(esona$timestamp)) - if(length(err)>0)esona = esona[-err,] + err = which(is.na(esona$timestamp)) + if(length(err)>0){ + cat(paste('Errors in time stamps for TRIP_NO--SETS'),unique(paste(esona$TRIP_ID[err],'--',esona$SET_NO[err],sep=""))) + esona = esona[-err,] + } esona$LATITUDE = format.lol(x = esona$LATITUDE) esona$LONGITUDE = format.lol(x = esona$LONGITUDE) #If specific years desired filter unwanted @@ -148,20 +185,34 @@ ilts.format.merge = function(update = TRUE, user = "", years = ""){ if(length(yind)>0)esona = esona[yind,] if(length(esona$timestamp)==0)stop("No data found for your year selection!") } - mini = get.oracle.table(tn = "FRAILC.MINILOG_TEMP") + #if(!use_local) { + # mini = get.oracle.table(tn = "FRAILC.MINILOG_TEMP", RODBC = use_RODBC) + # write.csv(mini,file=file.path(pkg.env$manual.archive, 'raw', 'MINILOG_TEMP.csv'),row.names = F) + # } + #if(use_local) mini = read.csv(file.path(pkg.env$manual.archive, 'raw', minilog.file)) + #rebuild datetime column as it is incorrect and order - mini$timestamp = lubridate::ymd_hms(paste(as.character(lubridate::date(mini$TDATE)), mini$TIME, sep=" "), tz="UTC" ) - mini = mini[ order(mini$timestamp , decreasing = FALSE ),] + #mini$timestamp = lubridate::ymd_hms(paste(as.character(lubridate::date(mini$TDATE)), mini$TIME, sep=" "), tz="UTC" ) + #mini = mini[ order(mini$timestamp , decreasing = FALSE ),] #seab = get.oracle.table(tn = "LOBSTER.ILTS_TEMPERATURE") - - seabf = get.oracle.table(tn = "FRAILC.ILTS_TEMPERATURE") + if(!use_local) { + seabf = get.oracle.table(tn = "frailc.ILTS_TEMPERATURE",RODBC = use_RODBC) + write.csv(seabf,file=file.path(pkg.env$manual.archive, 'raw', 'ILTS_TEMPERATURE.csv'),row.names = F) + } + if(use_local) seabf = read.csv(file.path(pkg.env$manual.archive, 'raw', seabird.file)) #rebuild datetime column as it is incorrect and order + seabf$UTCTIME[which(nchar(seabf$UTCTIME)==5)] = paste("0", seabf$UTCTIME[which(nchar(seabf$UTCTIME)==5)], sep="") + seabf$UTCTIME[which(nchar(seabf$UTCTIME)==4)] = paste("00", seabf$UTCTIME[which(nchar(seabf$UTCTIME)==4)], sep="") + seabf$UTCTIME[which(nchar(seabf$UTCTIME)==3)] = paste("000", seabf$UTCTIME[which(nchar(seabf$UTCTIME)==3)], sep="") + seabf$UTCTIME[which(nchar(seabf$UTCTIME)==2)] = paste("0000", seabf$UTCTIME[which(nchar(seabf$UTCTIME)==2)], sep="") + seabf$UTCTIME[which(nchar(seabf$UTCTIME)==1)] = paste("00000", seabf$UTCTIME[which(nchar(seabf$UTCTIME)==1)], sep="") seabf$timestamp = lubridate::ymd_hms(paste(as.character(lubridate::date(seabf$UTCDATE)), seabf$UTCTIME, sep=" "), tz="UTC" ) seabf = seabf[ order(seabf$timestamp , decreasing = FALSE ),] - #Loop through each esonar file to convert and merge with temp eson = split(esona, esona$TRIP_ID) + seabf$id = paste(seabf$TRIP_ID,seabf$SET_NO,sep="_") + for(i in 1:length(eson)){ if(cont){ #Condition fails if program exited trip = data.frame(eson[[i]]) @@ -170,57 +221,73 @@ ilts.format.merge = function(update = TRUE, user = "", years = ""){ if(cont){ #Condition fails if program exited set = data.frame(trip[[j]]) set = esonar2df(set) - + set$id = paste(set$Trip, set$Setno, sep="_") + sseab = subset(seabf, id == na.omit(unique(set$id))) + sseab = sseab[order(sseab$timestamp),] #Dont continue if update is false and station is already complete if((paste(unique(na.omit(set$Setno)), unique(na.omit(set$Trip)), sep = ".") %in% paste(current$station, current$trip, sep = ".")) & (update==FALSE)){ message(paste("Set:",unique(na.omit(set$Setno))," Trip:", unique(na.omit(set$Trip)), " Already added call with update = TRUE to redo.", sep = "")) } else{ - minisub = NULL - mini.ind.0 = which(mini$timestamp>set$timestamp[1])[1] - mini.ind.1 = which(mini$timestamp>set$timestamp[length(set$timestamp)])[1]-1 - if(!(is.na(mini.ind.0) | is.na(mini.ind.0))){ - minisub = mini[c(mini.ind.0:mini.ind.1),] - #### Only keep relevant data, If you encounter a minilog file - # depth, add that column to the following line to catch that case - if("depth" %in% names(minisub)){ - minisub = minisub[,which(names(minisub) %in% c("datetime", "TEMP_C", "depth"))] - names(minisub) = c("temperature","depth","timestamp") - } - else{ - minisub = minisub[,which(names(minisub) %in% c("datetime", "TEMP_C"))] - names(minisub) = c("temperature","timestamp") - } - } - - seabsub = NULL + seabsub = NULL #Get seabird indicies and extend ?? mins on either side so that depth profile isn't cut off - seab.ind.0 = which(seabf$timestamp>set$timestamp[1]-lubridate::minutes(15))[1] - seab.ind.1 = which(seabf$timestamp>set$timestamp[length(set$timestamp)]+lubridate::minutes(15))[1]-1 - - if(!(is.na(seab.ind.0) | is.na(seab.ind.0))){ - - seabsub = seabf[c(seab.ind.0:seab.ind.1),] - seabsub = seabsub[,which(names(seabsub) %in% c("timestamp", "TEMPC", "DEPTHM"))] + #seab.ind.0 = which(seabf$timestamp>set$timestamp[1]-lubridate::minutes(15))[1] + #seab.ind.1 = which(seabf$timestamp>set$timestamp[length(set$timestamp)]+lubridate::minutes(15))[1]-1 + #if(is.na(seab.ind.1)) seab.ind.1 = which(seabf$timestamp>set$timestamp[length(set$timestamp)]+lubridate::minutes(4))[1]-1 + + # if(!(is.na(seab.ind.0) | is.na(seab.ind.0))){ + # seabsub = seabf[c(seab.ind.0:seab.ind.1),] + seabsub = sseab + seabsub = seabsub[,which(names(seabsub) %in% c("timestamp", "TEMPC", "DEPTHM"))] names(seabsub) = c("temperature","depth","timestamp") + # } + if(is.null(seabsub)){ + print(paste("No temperature/depth file found for trip - set: ", unique(na.omit(set$Trip)), " - ", unique(na.omit(set$Setno)), sep="")) + next() } - - if(is.null(seabsub) && is.null(minisub))stop(paste("No temperature/depth file found for trip - set: ", unique(na.omit(set$Trip)), " - ", unique(na.omit(set$Setno)), sep="")) - if(is.null(seabsub)) seabsub = minisub + #if(is.null(seabsub)) seabsub = minisub #Remove depths = <0 #Not sure why but came accross stations with low depth values mixed in with real bottom depths. seabsub$depth[which(seabsub$depth <= 2)] = NA - + mergset = NULL #Merge sensor data. mergset = merge(seabsub, set, "timestamp", all = TRUE) #Build the full, unbroken timeseries and merge - timestamp = data.frame(seq(min(mergset$timestamp), max(mergset$timestamp), 1)) + timestamp = data.frame(seq(min(mergset$timestamp), max(mergset$timestamp), 1)) names(timestamp) = c("timestamp") mergset = merge(mergset, timestamp, "timestamp", all = TRUE) mergset$timestamp = lubridate::ymd_hms(as.character(mergset$timestamp), tz="UTC" ) #Find deepest point and extend possible data from that out to 20min on either side - aredown = mergset$timestamp[which(mergset$depth == max(mergset$depth, na.rm = T))] + + NoT = all(is.na(mergset$temperature)) + if(NoT) { + cat("\n",paste('No temperature info for Trip-Setno='),unique(paste(mergset$Trip,mergset$Setno,sep="-"))) + } + if(!NoT){ + if(temperature.plot){ + pdf(file.path(pkg.env$manual.archive, paste(unique(na.omit(mergset$Trip)), unique(na.omit(mergset$Setno)), 'Temperature.pdf',sep="."))) + with(subset(mergset, !is.na(temperature)),plot(timestamp,temperature, type='l')) + dev.off() + } + } + + NoDeps = all(is.na(mergset$depth)) + if(NoDeps) { + cat("\n",paste('No depth info for Trip-Setno='),unique(paste(mergset$Trip,mergset$Setno,sep="-"))) + write.csv(mergset,file=file.path(pkg.env$manual.archive, paste(unique(na.omit(mergset$Trip)), unique(na.omit(mergset$Setno)), 'csv',sep="."))) + + } + if(!NoDeps){ + aredown = mergset$timestamp[which(mergset$depth == max(mergset$depth, na.rm = T))] time.gate = list( t0=as.POSIXct(aredown)-lubridate::dminutes(20), t1=as.POSIXct(aredown)+lubridate::dminutes(20) ) + if(depth.plot){ + pdf(file.path(pkg.env$manual.archive, paste(unique(na.omit(mergset$Trip)), unique(na.omit(mergset$Setno)), 'pdf',sep="."))) + with(subset(mergset, !is.na(depth)),plot(timestamp,depth, type='l')) + dev.off() + write.csv(mergset,file=file.path(pkg.env$manual.archive, paste(unique(na.omit(mergset$Trip)), unique(na.omit(mergset$Setno)), 'csv',sep="."))) + next() + } + # Build the variables need for the proper execution of the bottom contact function from # the netmensuration package bcp = list( @@ -310,6 +377,7 @@ ilts.format.merge = function(update = TRUE, user = "", years = ""){ } iltsStats[[paste(unique(na.omit(set$Trip)), unique(na.omit(set$Setno)),sep=".")]] = bc + } #end if no depth }#END Update clause } }#END Set subset diff --git a/Readme.txt b/Readme.txt index e75ceee..9f95e3e 100644 --- a/Readme.txt +++ b/Readme.txt @@ -5,7 +5,7 @@ To Install library(devtools) install_github("jae0/netmensuration") install_github("brent0/ILTS.sensor") - install_github("brent0/ILTS.sensor", INSTALL_opts=c("--no-multiarch")) + install_github(""brent0/ILTS.sensor"", INSTALL_opts=c("--no-multiarch")) To Develop @@ -13,17 +13,31 @@ To Develop enter this url https://github.com/brent0/ILTS.sensor.git let me know so I can add your git account as contributor or you can fork the code for your own project. -To Avoid entering project variables in the dialog prompts, set the following in your Rprofile.site file +To Avoid entering project variables in the dialog prompts, set the following in your Rprofile.site file or to the top of your script. - oracle.snowcrab.server = "ptran" + oracle.snowcrab.server = "ptran" oracle.snowcrab.user = "your oracle username" oracle.snowcrab.password = "your oracle password" - bio.datadirectory.ilts = file.path( "C:", "bio.data", "ilts") replace with desired path + bio.datadirectory.ilts = file.path( "C:", "bio.data", "ilts") #replace with desired path + dir.create(file.path(bio.datadirectory.ilts,'raw'),showWarnings = F) + +You can work using ODBC connections through ROracle, RODBC or work locally. To work remotely you need select access to the following tables: + + FRAILC.ILTS_SENSORS_TEMP FRAILC.MINILOG_TEMP and FRAILC.ILTS_TEMPERATURE. + +If you run the code through the ODBC connections you will automatically create local copies for use later with use_local=T. + +If you are working on local copies you need to create a folder in bio.datadirectory.ilts called 'raw'where the output of the ILTS_SENSORS_TEMP.csv, MINILOG_TEMP.csv and ILTS_TEMPERATURE.csv reside. To start determining bottom touchdown, or to redo any previously determined sets: - ilts.format.merge(update = TRUE, user = "brent" ) Set user to whatever you like. +ilts.format.merge(update = TRUE, user = "brent", use_RODBC = F, use_local = F ) #Set user to whatever you like and indicate how you want to access the data. + +If you want to get only the depth and timestamp plots for each set run: + +ilts.format.merge(update = TRUE, user = "brent", use_RODBC = F, use_local = F,depth.only.plot=T ) + To stop just click the top right x in the plot window at any time @@ -52,4 +66,4 @@ ERRORS L04092019 set 4 did plot before did when testing expanded plot - + \ No newline at end of file