Skip to content

Adding in functionality for RODBC and use_local files. Also added som… #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AMCOOK wants to merge 12 commits into
base: master
Choose a base branch
from
Open

Adding in functionality for RODBC and use_local files. Also added som… #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
de1596f
Adding in functionality for RODBC and use_local files. Also added som…
AMCOOK May 8, 2020
704da1e
Removing ROracle import
May 12, 2020
41d08c8
dir.create statements
May 12, 2020
c6ab287
added depth only plots to the args
AMCOOK May 19, 2020
adacc52
fixes
AMCOOK May 19, 2020
3e088ff
fixes
AMCOOK May 19, 2020
8f9420a
Column naming conventions flawed'
May 20, 2020
c142b38
fix
May 20, 2020
9e98142
Date fixes
May 20, 2020
223d432
Removing call to minilog, and cleaning time stamps from seabird
May 21, 2020
dade784
Writing all output
May 22, 2020
e7f5e02
Adding temperature plots
May 29, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions DESCRIPTION
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ Imports:
tcltk,
gWidgets2,
gWidgets2tcltk,
ROracle,
DBI,
netmensuration,
lubridate,
stringr
stringr,
RODBC
2 changes: 1 addition & 1 deletion NAMESPACE
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ export(get.oracle.table)
export(ilts.format.merge)
export(init.project.vars)
import(DBI)
import(ROracle)
import(gWidgets2)
import(gWidgets2tcltk)
import(lubridate)
import(netmensuration)
import(stringr)
import(tcltk)
import(RODBC)
176 changes: 122 additions & 54 deletions R/ilts.operations.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,18 @@ assign('manual.archive', "", pkg.env)
assign('oracle.server', "", pkg.env)
assign('oracle.user', "", pkg.env)
assign('oracle.password', "", pkg.env)
options(stringsAsFactors = F)

#' @title rename.df
#' @description renaming columns
#' @export
rename.df = function(x, n0, n1) {
if(!length(n0)== length(n1)) stop('length of names and renames need to be the same length')
for(i in 1:length(n0)){
names(x)[which(names(x)==n0[i])] = n1[i]
}
return(x)
}

#' @title init.project.vars
#' @description Set global database variables by user account
Expand All @@ -12,12 +24,13 @@ assign('oracle.password', "", pkg.env)
init.project.vars = function() {
if(exists("bio.datadirectory.ilts")){
assign('manual.archive', bio.datadirectory.ilts , pkg.env)
}
else{
}
if(!exists("bio.datadirectory.ilts")){
print('A window will open that you need to choose the working folder.')
assign('manual.archive', gWidgets2::gfile(text = "Select project work directory", type = "selectdir"), pkg.env)

}

}
dir.create(pkg.env$manual.archive, showWarnings = F)
#Take from snowcrab users .Rprofile.site
if(exists("oracle.snowcrab.server")){
assign('oracle.server', oracle.snowcrab.server, pkg.env)
Expand Down Expand Up @@ -47,8 +60,9 @@ init.project.vars = function() {
#' @return dataframe
#' @export
esonar2df = function(esonar = NULL) {
names(esonar)
colnames(esonar) = c("CPUDateTime","GPSDate","GPSTime","Latitude","Longitude","Speed","Heading","Validity","TransducerName","SensorName","SensorValue","ErrorCode","Hydrophone","SignalStrength", "setno", "latedit", "trip", "datetime")

esonar = rename.df(esonar, c( 'CPUDATEANDTIME','GPSTIME','LATITUDE','LONGITUDE','SPEED','HEADING','VALIDITY','TRANSDUCERNAME','SENSORNAME','SENSORVALUE','ERRORCODE','HYDROPHONE','SIGNALSTRENGTH','SET_NO','LATEDIT','TRIP_ID','GPSDATE','timestamp'),
c("CPUDateTime","GPSTime","Latitude","Longitude","Speed","Heading","Validity","TransducerName","SensorName","SensorValue","ErrorCode","Hydrophone","SignalStrength", "setno", "latedit", "trip", "GPSDate","datetime"))

esonar$primary = NA #Headline
esonar$secondary = NA #Is nothing but may need in file
Expand All @@ -74,27 +88,34 @@ esonar2df = function(esonar = NULL) {
esonar$Validity = NULL
esonar$ErrorCode = NULL
esonar$Heading = NULL
colnames(esonar) = c("Date","Time","Latitude","Longitude","Speed", "Setno", "latedit","Trip","timestamp", "Primary","Secondary","WingSpread","Roll", "Pitch")
esonar = rename.df(esonar, c('GPSTime','Latitude','Longitude','Speed','setno','latedit','trip','GPSDate','datetime','primary','secondary','wingspread','STBDRoll','STBDPitch'),
c("Time","Latitude","Longitude","Speed", "Setno", "latedit","Trip","Date","timestamp", "Primary","Secondary","WingSpread","Roll", "Pitch"))

return(esonar)
}

#' @title get.acoustic.releases
#' @title get.oracle.table
#' @description Get data from Oracle Database table
#' @import ROracle DBI
#' @import ROracle DBI RODBC
#' @param tn tablename to get
#' @param oracle.user your oracle username
#' @param oracle.password your oracle password
#' @return dataframe
#' @export
get.oracle.table = function(tn = "",server = pkg.env$oracle.server, user =pkg.env$oracle.user, password = pkg.env$oracle.password){
get.oracle.table = function(tn = "",server = pkg.env$oracle.server, user =pkg.env$oracle.user, password = pkg.env$oracle.password, RODBC=F){
if(tn == "")stop("You must provide a tablename to 'get.oracle.table'!!")

if(!RODBC){
drv <- dbDriver("Oracle")
con <- ROracle::dbConnect(drv, username = user, password = password, dbname = server)
res <- ROracle::dbSendQuery(con, paste("select * from ", tn, sep=""))
res <- fetch(res)
ROracle::dbDisconnect(con)
}
if(RODBC) {
drv = odbcConnect(dsn = server ,uid = user, pwd = password)
res = sqlQuery(drv,paste("select * from ", tn,";", sep=""))
odbcCloseAll()
}
return(res)
}

Expand All @@ -107,9 +128,10 @@ get.oracle.table = function(tn = "",server = pkg.env$oracle.server, user =pkg.en
#' @import netmensuration lubridate
#' @return list of lists. Format (top to bottom) year-set-data
#' @export
ilts.format.merge = function(update = TRUE, user = "", years = ""){
ilts.format.merge = function(update = TRUE, user = "", years = "", use_RODBC=F, use_local=F, depth.plot=F,temperature.plot=F, sensor.file='ILTS_SENSORS_TEMP.csv', minilog.file = 'MINILOG_TEMP.csv', seabird.file = 'ILTS_TEMPERATURE.csv'){
#Set up database server, user and password
init.project.vars()
options(stringsAsFactors=F)

cont=TRUE
if(user == "")stop("You must call this function with a user. exa. ilts.format.merge(update = TRUE, user = 'John'" )
Expand All @@ -133,12 +155,27 @@ ilts.format.merge = function(update = TRUE, user = "", years = ""){
plotdata = F #Alternative plotting that we do not require

#Pull in the sensor data, this will be formatted and looped thru by trip then set.
esona = get.oracle.table(tn = "FRAILC.ILTS_SENSORS_TEMP")
dir.create(file.path(pkg.env$manual.archive,'raw'), showWarnings = F)

if(!use_local) { esona = get.oracle.table(tn = "FRAILC.ILTS_SENSORS_TEMP", RODBC=use_RODBC)
write.csv(esona,file=file.path(pkg.env$manual.archive, 'raw', 'ILTS_SENSORS_TEMP.csv'),row.names = F)
}
if(use_local) esona = read.csv(file.path(pkg.env$manual.archive, 'raw', sensor.file))

esona$GPSTIME[which(nchar(esona$GPSTIME)==5)] = paste("0", esona$GPSTIME[which(nchar(esona$GPSTIME)==5)], sep="")
esona$GPSTIME[which(nchar(esona$GPSTIME)==5)] = paste("0", esona$GPSTIME[which(nchar(esona$GPSTIME)==5)], sep="")
esona$GPSTIME[which(nchar(esona$GPSTIME)==4)] = paste("00", esona$GPSTIME[which(nchar(esona$GPSTIME)==4)], sep="")
esona$GPSTIME[which(nchar(esona$GPSTIME)==3)] = paste("000", esona$GPSTIME[which(nchar(esona$GPSTIME)==3)], sep="")
esona$GPSTIME[which(nchar(esona$GPSTIME)==2)] = paste("0000", esona$GPSTIME[which(nchar(esona$GPSTIME)==2)], sep="")
esona$GPSTIME[which(nchar(esona$GPSTIME)==1)] = paste("00000", esona$GPSTIME[which(nchar(esona$GPSTIME)==1)], sep="")

esona$timestamp = lubridate::ymd_hms(paste(as.character(lubridate::date(esona$GPSDATE)), esona$GPSTIME, sep=" "), tz="UTC" )
esona = esona[ order(esona$timestamp , decreasing = FALSE ),]
err = which(is.na(esona$timestamp))
if(length(err)>0)esona = esona[-err,]
err = which(is.na(esona$timestamp))
if(length(err)>0){
cat(paste('Errors in time stamps for TRIP_NO--SETS'),unique(paste(esona$TRIP_ID[err],'--',esona$SET_NO[err],sep="")))
esona = esona[-err,]
}
esona$LATITUDE = format.lol(x = esona$LATITUDE)
esona$LONGITUDE = format.lol(x = esona$LONGITUDE)
#If specific years desired filter unwanted
Expand All @@ -148,20 +185,34 @@ ilts.format.merge = function(update = TRUE, user = "", years = ""){
if(length(yind)>0)esona = esona[yind,]
if(length(esona$timestamp)==0)stop("No data found for your year selection!")
}
mini = get.oracle.table(tn = "FRAILC.MINILOG_TEMP")
#if(!use_local) {
# mini = get.oracle.table(tn = "FRAILC.MINILOG_TEMP", RODBC = use_RODBC)
# write.csv(mini,file=file.path(pkg.env$manual.archive, 'raw', 'MINILOG_TEMP.csv'),row.names = F)
# }
#if(use_local) mini = read.csv(file.path(pkg.env$manual.archive, 'raw', minilog.file))

#rebuild datetime column as it is incorrect and order
mini$timestamp = lubridate::ymd_hms(paste(as.character(lubridate::date(mini$TDATE)), mini$TIME, sep=" "), tz="UTC" )
mini = mini[ order(mini$timestamp , decreasing = FALSE ),]
#mini$timestamp = lubridate::ymd_hms(paste(as.character(lubridate::date(mini$TDATE)), mini$TIME, sep=" "), tz="UTC" )
#mini = mini[ order(mini$timestamp , decreasing = FALSE ),]

#seab = get.oracle.table(tn = "LOBSTER.ILTS_TEMPERATURE")

seabf = get.oracle.table(tn = "FRAILC.ILTS_TEMPERATURE")
if(!use_local) {
seabf = get.oracle.table(tn = "frailc.ILTS_TEMPERATURE",RODBC = use_RODBC)
write.csv(seabf,file=file.path(pkg.env$manual.archive, 'raw', 'ILTS_TEMPERATURE.csv'),row.names = F)
}
if(use_local) seabf = read.csv(file.path(pkg.env$manual.archive, 'raw', seabird.file))
#rebuild datetime column as it is incorrect and order
seabf$UTCTIME[which(nchar(seabf$UTCTIME)==5)] = paste("0", seabf$UTCTIME[which(nchar(seabf$UTCTIME)==5)], sep="")
seabf$UTCTIME[which(nchar(seabf$UTCTIME)==4)] = paste("00", seabf$UTCTIME[which(nchar(seabf$UTCTIME)==4)], sep="")
seabf$UTCTIME[which(nchar(seabf$UTCTIME)==3)] = paste("000", seabf$UTCTIME[which(nchar(seabf$UTCTIME)==3)], sep="")
seabf$UTCTIME[which(nchar(seabf$UTCTIME)==2)] = paste("0000", seabf$UTCTIME[which(nchar(seabf$UTCTIME)==2)], sep="")
seabf$UTCTIME[which(nchar(seabf$UTCTIME)==1)] = paste("00000", seabf$UTCTIME[which(nchar(seabf$UTCTIME)==1)], sep="")
seabf$timestamp = lubridate::ymd_hms(paste(as.character(lubridate::date(seabf$UTCDATE)), seabf$UTCTIME, sep=" "), tz="UTC" )
seabf = seabf[ order(seabf$timestamp , decreasing = FALSE ),]

#Loop through each esonar file to convert and merge with temp
eson = split(esona, esona$TRIP_ID)
seabf$id = paste(seabf$TRIP_ID,seabf$SET_NO,sep="_")

for(i in 1:length(eson)){
if(cont){ #Condition fails if program exited
trip = data.frame(eson[[i]])
Expand All @@ -170,57 +221,73 @@ ilts.format.merge = function(update = TRUE, user = "", years = ""){
if(cont){ #Condition fails if program exited
set = data.frame(trip[[j]])
set = esonar2df(set)

set$id = paste(set$Trip, set$Setno, sep="_")
sseab = subset(seabf, id == na.omit(unique(set$id)))
sseab = sseab[order(sseab$timestamp),]
#Dont continue if update is false and station is already complete

if((paste(unique(na.omit(set$Setno)), unique(na.omit(set$Trip)), sep = ".") %in% paste(current$station, current$trip, sep = ".")) & (update==FALSE)){
message(paste("Set:",unique(na.omit(set$Setno))," Trip:", unique(na.omit(set$Trip)), " Already added call with update = TRUE to redo.", sep = ""))
} else{
minisub = NULL
mini.ind.0 = which(mini$timestamp>set$timestamp[1])[1]
mini.ind.1 = which(mini$timestamp>set$timestamp[length(set$timestamp)])[1]-1
if(!(is.na(mini.ind.0) | is.na(mini.ind.0))){
minisub = mini[c(mini.ind.0:mini.ind.1),]
#### Only keep relevant data, If you encounter a minilog file
# depth, add that column to the following line to catch that case
if("depth" %in% names(minisub)){
minisub = minisub[,which(names(minisub) %in% c("datetime", "TEMP_C", "depth"))]
names(minisub) = c("temperature","depth","timestamp")
}
else{
minisub = minisub[,which(names(minisub) %in% c("datetime", "TEMP_C"))]
names(minisub) = c("temperature","timestamp")
}
}

seabsub = NULL
seabsub = NULL
#Get seabird indicies and extend ?? mins on either side so that depth profile isn't cut off
seab.ind.0 = which(seabf$timestamp>set$timestamp[1]-lubridate::minutes(15))[1]
seab.ind.1 = which(seabf$timestamp>set$timestamp[length(set$timestamp)]+lubridate::minutes(15))[1]-1

if(!(is.na(seab.ind.0) | is.na(seab.ind.0))){

seabsub = seabf[c(seab.ind.0:seab.ind.1),]
seabsub = seabsub[,which(names(seabsub) %in% c("timestamp", "TEMPC", "DEPTHM"))]
#seab.ind.0 = which(seabf$timestamp>set$timestamp[1]-lubridate::minutes(15))[1]
#seab.ind.1 = which(seabf$timestamp>set$timestamp[length(set$timestamp)]+lubridate::minutes(15))[1]-1
#if(is.na(seab.ind.1)) seab.ind.1 = which(seabf$timestamp>set$timestamp[length(set$timestamp)]+lubridate::minutes(4))[1]-1

# if(!(is.na(seab.ind.0) | is.na(seab.ind.0))){
# seabsub = seabf[c(seab.ind.0:seab.ind.1),]
seabsub = sseab
seabsub = seabsub[,which(names(seabsub) %in% c("timestamp", "TEMPC", "DEPTHM"))]
names(seabsub) = c("temperature","depth","timestamp")
# }
if(is.null(seabsub)){
print(paste("No temperature/depth file found for trip - set: ", unique(na.omit(set$Trip)), " - ", unique(na.omit(set$Setno)), sep=""))
next()
}

if(is.null(seabsub) && is.null(minisub))stop(paste("No temperature/depth file found for trip - set: ", unique(na.omit(set$Trip)), " - ", unique(na.omit(set$Setno)), sep=""))
if(is.null(seabsub)) seabsub = minisub
#if(is.null(seabsub)) seabsub = minisub
#Remove depths = <0 #Not sure why but came accross stations with low depth values mixed in with real bottom depths.
seabsub$depth[which(seabsub$depth <= 2)] = NA

mergset = NULL
#Merge sensor data.
mergset = merge(seabsub, set, "timestamp", all = TRUE)
#Build the full, unbroken timeseries and merge
timestamp = data.frame(seq(min(mergset$timestamp), max(mergset$timestamp), 1))
timestamp = data.frame(seq(min(mergset$timestamp), max(mergset$timestamp), 1))
names(timestamp) = c("timestamp")
mergset = merge(mergset, timestamp, "timestamp", all = TRUE)
mergset$timestamp = lubridate::ymd_hms(as.character(mergset$timestamp), tz="UTC" )
#Find deepest point and extend possible data from that out to 20min on either side
aredown = mergset$timestamp[which(mergset$depth == max(mergset$depth, na.rm = T))]

NoT = all(is.na(mergset$temperature))
if(NoT) {
cat("\n",paste('No temperature info for Trip-Setno='),unique(paste(mergset$Trip,mergset$Setno,sep="-")))
}
if(!NoT){
if(temperature.plot){
pdf(file.path(pkg.env$manual.archive, paste(unique(na.omit(mergset$Trip)), unique(na.omit(mergset$Setno)), 'Temperature.pdf',sep=".")))
with(subset(mergset, !is.na(temperature)),plot(timestamp,temperature, type='l'))
dev.off()
}
}

NoDeps = all(is.na(mergset$depth))
if(NoDeps) {
cat("\n",paste('No depth info for Trip-Setno='),unique(paste(mergset$Trip,mergset$Setno,sep="-")))
write.csv(mergset,file=file.path(pkg.env$manual.archive, paste(unique(na.omit(mergset$Trip)), unique(na.omit(mergset$Setno)), 'csv',sep=".")))

}
if(!NoDeps){
aredown = mergset$timestamp[which(mergset$depth == max(mergset$depth, na.rm = T))]
time.gate = list( t0=as.POSIXct(aredown)-lubridate::dminutes(20), t1=as.POSIXct(aredown)+lubridate::dminutes(20) )

if(depth.plot){
pdf(file.path(pkg.env$manual.archive, paste(unique(na.omit(mergset$Trip)), unique(na.omit(mergset$Setno)), 'pdf',sep=".")))
with(subset(mergset, !is.na(depth)),plot(timestamp,depth, type='l'))
dev.off()
write.csv(mergset,file=file.path(pkg.env$manual.archive, paste(unique(na.omit(mergset$Trip)), unique(na.omit(mergset$Setno)), 'csv',sep=".")))
next()
}

# Build the variables need for the proper execution of the bottom contact function from
# the netmensuration package
bcp = list(
Expand Down Expand Up @@ -310,6 +377,7 @@ ilts.format.merge = function(update = TRUE, user = "", years = ""){
}

iltsStats[[paste(unique(na.omit(set$Trip)), unique(na.omit(set$Setno)),sep=".")]] = bc
} #end if no depth
}#END Update clause
}
}#END Set subset
Expand Down
26 changes: 20 additions & 6 deletions Readme.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,25 +5,39 @@ To Install
library(devtools)
install_github("jae0/netmensuration")
install_github("brent0/ILTS.sensor")
install_github("brent0/ILTS.sensor", INSTALL_opts=c("--no-multiarch"))
install_github(""brent0/ILTS.sensor"", INSTALL_opts=c("--no-multiarch"))

To Develop

In RStudio go to file -> new project -> version controll
enter this url https://github.com/brent0/ILTS.sensor.git
let me know so I can add your git account as contributor or you can fork the code for your own project.

To Avoid entering project variables in the dialog prompts, set the following in your Rprofile.site file
To Avoid entering project variables in the dialog prompts, set the following in your Rprofile.site file or to the top of your script.

oracle.snowcrab.server = "ptran"
oracle.snowcrab.server = "ptran"
oracle.snowcrab.user = "your oracle username"
oracle.snowcrab.password = "your oracle password"
bio.datadirectory.ilts = file.path( "C:", "bio.data", "ilts") replace with desired path
bio.datadirectory.ilts = file.path( "C:", "bio.data", "ilts") #replace with desired path
dir.create(file.path(bio.datadirectory.ilts,'raw'),showWarnings = F)

You can work using ODBC connections through ROracle, RODBC or work locally. To work remotely you need select access to the following tables:

FRAILC.ILTS_SENSORS_TEMP FRAILC.MINILOG_TEMP and FRAILC.ILTS_TEMPERATURE.

If you run the code through the ODBC connections you will automatically create local copies for use later with use_local=T.

If you are working on local copies you need to create a folder in bio.datadirectory.ilts called 'raw'where the output of the ILTS_SENSORS_TEMP.csv, MINILOG_TEMP.csv and ILTS_TEMPERATURE.csv reside.


To start determining bottom touchdown, or to redo any previously determined sets:

ilts.format.merge(update = TRUE, user = "brent" ) Set user to whatever you like.
ilts.format.merge(update = TRUE, user = "brent", use_RODBC = F, use_local = F ) #Set user to whatever you like and indicate how you want to access the data.

If you want to get only the depth and timestamp plots for each set run:

ilts.format.merge(update = TRUE, user = "brent", use_RODBC = F, use_local = F,depth.only.plot=T )


To stop just click the top right x in the plot window at any time

Expand Down Expand Up @@ -52,4 +66,4 @@ ERRORS
L04092019 set 4 did plot before did when testing expanded plot